From 4dd9409f951bd0d92a5b703bc970f98946abc678 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 19 May 2025 10:20:42 +0200
Subject: [PATCH 01/22] WIP: Read zarr agglomerate files

---
 .../services/AgglomerateService.scala         | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index ed970194131..2e0bfbd8b5e 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -24,7 +24,18 @@ import scala.annotation.tailrec
 import scala.collection.compat.immutable.ArraySeq
 import scala.concurrent.duration.DurationInt
 
-class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
+class ZarrAgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
+  def applyAgglomerateHdf5(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] = tryo(data)
+
+}
+
+class Hdf5AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
+  // TODO
+}
+
+class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateService: ZarrAgglomerateService)
+    extends DataConverter
+    with LazyLogging {
   private val agglomerateDir = "agglomerates"
   private val agglomerateFileExtension = "hdf5"
   private val datasetName = "/segment_to_agglomerate"
@@ -47,7 +58,12 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
       .toSet
   }
 
-  def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] = tryo {
+  def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] =
+    if (true) {
+      zarrAgglomerateService.applyAgglomerateHdf5(request)(data)
+    } else applyAgglomerateHdf5(request)(data)
+
+  private def applyAgglomerateHdf5(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] = tryo {
 
     val agglomerateFileKey = AgglomerateFileKey.fromDataRequest(request)
 

From 380bd69ed47cc23bafca221798a714ca0d98fae7 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 19 May 2025 11:27:05 +0200
Subject: [PATCH 02/22] zarr group path

---
 .../datastore/services/AgglomerateService.scala     | 13 ++++++++++++-
 .../datastore/storage/AgglomerateFileCache.scala    |  8 ++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 2e0bfbd8b5e..3f84333d6e4 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -25,7 +25,18 @@ import scala.collection.compat.immutable.ArraySeq
 import scala.concurrent.duration.DurationInt
 
 class ZarrAgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
-  def applyAgglomerateHdf5(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] = tryo(data)
+  private val dataBaseDir = Paths.get(config.Datastore.baseDirectory)
+  private val agglomerateDir = "agglomerates"
+  private val agglomerateFileExtension = ""
+
+  def applyAgglomerateHdf5(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] = tryo {
+
+    val agglomerateFileKey = AgglomerateFileKey.fromDataRequest(request)
+
+    val zarrGroupPath = agglomerateFileKey.zarrGroupPath(dataBaseDir, agglomerateDir)
+
+    data
+  }
 
 }
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
index 018bd27e9f9..9a9e2106ba4 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
@@ -32,6 +32,14 @@ case class AgglomerateFileKey(
       .resolve(layerName)
       .resolve(agglomerateDir)
       .resolve(s"$mappingName.$agglomerateFileExtension")
+
+  def zarrGroupPath(dataBaseDir: Path, agglomerateDir: String): Path =
+    dataBaseDir
+      .resolve(organizationId)
+      .resolve(datasetDirectoryName)
+      .resolve(layerName)
+      .resolve(agglomerateDir)
+      .resolve(mappingName)
 }
 
 object AgglomerateFileKey {

From 7fb643fe19ad2be8037b34bcb85f91cc7f1246f0 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 19 May 2025 13:24:57 +0200
Subject: [PATCH 03/22] test reading from zarr array

---
 .../datastore/controllers/Application.scala   | 22 ++++--
 .../datastore/datareaders/DatasetArray.scala  |  5 +-
 .../datareaders/zarr3/Zarr3Array.scala        |  7 +-
 .../services/AgglomerateService.scala         | 68 ++++++++++++++++---
 .../services/BinaryDataService.scala          |  2 +-
 .../services/mesh/AdHocMeshService.scala      | 10 +--
 ....scalableminds.webknossos.datastore.routes |  1 +
 7 files changed, 91 insertions(+), 24 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
index 693b917d7dd..3ede25cb42f 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
@@ -3,8 +3,13 @@ package com.scalableminds.webknossos.datastore.controllers
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.helpers.NativeBucketScanner
-import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
-import com.scalableminds.webknossos.datastore.services.ApplicationHealthService
+import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, ElementClass}
+import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
+import com.scalableminds.webknossos.datastore.services.{
+  AgglomerateService,
+  ApplicationHealthService,
+  ZarrAgglomerateService
+}
 import com.scalableminds.webknossos.datastore.storage.DataStoreRedisStore
 import net.liftweb.common.Box.tryo
 
@@ -13,8 +18,9 @@ import play.api.mvc.{Action, AnyContent}
 
 import scala.concurrent.ExecutionContext
 
-class Application @Inject()(redisClient: DataStoreRedisStore, applicationHealthService: ApplicationHealthService)(
-    implicit ec: ExecutionContext)
+class Application @Inject()(redisClient: DataStoreRedisStore,
+                            applicationHealthService: ApplicationHealthService,
+                            agglomerateService: ZarrAgglomerateService)(implicit ec: ExecutionContext)
     extends Controller {
 
   override def allowRemoteOrigin: Boolean = true
@@ -34,6 +40,14 @@ class Application @Inject()(redisClient: DataStoreRedisStore, applicationHealthS
     }
   }
 
+  def testAgglomerateZarr: Action[AnyContent] = Action.async { implicit request =>
+    log() {
+      for {
+        data <- agglomerateService.readFromSegmentToAgglomerate
+      } yield Ok(s"got ${data.length} bytes")
+    }
+  }
+
   // Test that the NativeBucketScanner works.
   // The result is stored in a val because we expect that this continues to work if it works on startup.
   private lazy val testNativeBucketScanner = tryo {
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
index 5c17139b01e..2a00dd70a7b 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
@@ -116,8 +116,9 @@ class DatasetArray(vaultPath: VaultPath,
   }
 
   // returns byte array in fortran-order with little-endian values
-  private def readBytes(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext,
-                                                               tc: TokenContext): Fox[Array[Byte]] =
+  // TODO should possibly be private again
+  def readBytes(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext,
+                                                       tc: TokenContext): Fox[Array[Byte]] =
     for {
       typedMultiArray <- readAsFortranOrder(shape, offset)
       asBytes <- BytesConverter.toByteArray(typedMultiArray, header.resolvedDataType, ByteOrder.LITTLE_ENDIAN).toFox
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3Array.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3Array.scala
index 40ea0c2e934..5a7d0c7b807 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3Array.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3Array.scala
@@ -22,10 +22,10 @@ object Zarr3Array extends LazyLogging with FoxImplicits {
            channelIndex: Option[Int],
            additionalAxes: Option[Seq[AdditionalAxis]],
            sharedChunkContentsCache: AlfuCache[String, MultiArray])(implicit ec: ExecutionContext,
-                                                                    tc: TokenContext): Fox[Zarr3Array] =
+                                                                    tc: TokenContext): Fox[Zarr3Array] = {
+    val headerPath = path / Zarr3ArrayHeader.FILENAME_ZARR_JSON
     for {
-      headerBytes <- (path / Zarr3ArrayHeader.FILENAME_ZARR_JSON)
-        .readBytes() ?~> s"Could not read header at ${Zarr3ArrayHeader.FILENAME_ZARR_JSON}"
+      headerBytes <- headerPath.readBytes() ?~> s"Could not read header at $headerPath"
       header <- JsonHelper.parseAs[Zarr3ArrayHeader](headerBytes).toFox ?~> "Could not parse array header"
       array <- tryo(
         new Zarr3Array(path,
@@ -37,6 +37,7 @@ object Zarr3Array extends LazyLogging with FoxImplicits {
                        additionalAxes,
                        sharedChunkContentsCache)).toFox ?~> "Could not open zarr3 array"
     } yield array
+  }
 }
 
 class Zarr3Array(vaultPath: VaultPath,
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 3f84333d6e4..e098be3c369 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -1,41 +1,89 @@
 package com.scalableminds.webknossos.datastore.services
 
 import ch.systemsx.cisd.hdf5._
+import com.scalableminds.util.accesscontext.TokenContext
+import com.scalableminds.util.cache.AlfuCache
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.io.PathUtils
 import com.scalableminds.util.time.Instant
+import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.AgglomerateGraph.{AgglomerateEdge, AgglomerateGraph}
 import com.scalableminds.webknossos.datastore.DataStoreConfig
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, SkeletonTracing, Tree, TreeTypeProto}
+import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3Array
 import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, SkeletonTracingDefaults}
-import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
+import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, ElementClass}
 import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
 import com.scalableminds.webknossos.datastore.storage._
 import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.{Box, Failure, Full}
 import net.liftweb.common.Box.tryo
 import org.apache.commons.io.FilenameUtils
+import ucar.ma2.{Array => MultiArray}
 
+import java.net.URI
 import java.nio._
 import java.nio.file.{Files, Paths}
 import javax.inject.Inject
 import scala.annotation.tailrec
 import scala.collection.compat.immutable.ArraySeq
+import scala.concurrent.ExecutionContext
 import scala.concurrent.duration.DurationInt
 
-class ZarrAgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
+class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService: DataVaultService)
+    extends DataConverter
+    with LazyLogging {
   private val dataBaseDir = Paths.get(config.Datastore.baseDirectory)
   private val agglomerateDir = "agglomerates"
   private val agglomerateFileExtension = ""
 
-  def applyAgglomerateHdf5(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] = tryo {
+  private lazy val sharedChunkContentsCache: AlfuCache[String, MultiArray] = {
+    // Used by DatasetArray-based datasets. Measure item weight in kilobytes because the weigher can only return int, not long
+
+    val maxSizeKiloBytes = Math.floor(config.Datastore.Cache.ImageArrayChunks.maxSizeBytes.toDouble / 1000.0).toInt
+
+    def cacheWeight(key: String, arrayBox: Box[MultiArray]): Int =
+      arrayBox match {
+        case Full(array) =>
+          (array.getSizeBytes / 1000L).toInt
+        case _ => 0
+      }
+
+    AlfuCache(maxSizeKiloBytes, weighFn = Some(cacheWeight))
+  }
+
+  def readFromSegmentToAgglomerate(implicit ec: ExecutionContext): Fox[Array[Byte]] = {
+    val zarrGroupPath =
+      dataBaseDir
+        .resolve("sample_organization/test-agglomerate-file-zarr/segmentation/agglomerates/agglomerate_view_5")
+        .toAbsolutePath
+    for {
+      groupVaultPath <- dataVaultService.getVaultPath(RemoteSourceDescriptor(new URI(s"file://$zarrGroupPath"), None))
+      segmentToAgglomeratePath = groupVaultPath / "segment_to_agglomerate"
+      zarrArray <- Zarr3Array.open(segmentToAgglomeratePath,
+                                   DataSourceId("zarr", "test"),
+                                   "layer",
+                                   None,
+                                   None,
+                                   None,
+                                   sharedChunkContentsCache)(ec, TokenContext(None))
+      read <- zarrArray.readBytes(Array(5), Array(0))(ec, TokenContext(None))
+      _ = logger.info(s"read ${read.length} bytes from agglomerate file")
+    } yield read
+  }
+
+  def applyAgglomerateHdf5(request: DataServiceDataRequest)(data: Array[Byte])(
+      implicit ec: ExecutionContext): Fox[Array[Byte]] = {
 
     val agglomerateFileKey = AgglomerateFileKey.fromDataRequest(request)
 
-    val zarrGroupPath = agglomerateFileKey.zarrGroupPath(dataBaseDir, agglomerateDir)
+    val zarrGroupPath = agglomerateFileKey.zarrGroupPath(dataBaseDir, agglomerateDir).toAbsolutePath
+
+    for {
+      _ <- readFromSegmentToAgglomerate
+    } yield data
 
-    data
   }
 
 }
@@ -46,7 +94,8 @@ class Hdf5AgglomerateService @Inject()(config: DataStoreConfig) extends DataConv
 
 class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateService: ZarrAgglomerateService)
     extends DataConverter
-    with LazyLogging {
+    with LazyLogging
+    with FoxImplicits {
   private val agglomerateDir = "agglomerates"
   private val agglomerateFileExtension = "hdf5"
   private val datasetName = "/segment_to_agglomerate"
@@ -66,13 +115,14 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
       }
       .toOption
       .getOrElse(Nil)
-      .toSet
+      .toSet ++ Set("agglomerate_view_5") // TODO
   }
 
-  def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] =
+  def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(
+      implicit ec: ExecutionContext): Fox[Array[Byte]] =
     if (true) {
       zarrAgglomerateService.applyAgglomerateHdf5(request)(data)
-    } else applyAgglomerateHdf5(request)(data)
+    } else applyAgglomerateHdf5(request)(data).toFox
 
   private def applyAgglomerateHdf5(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] = tryo {
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
index 55837d02665..6d82ad62077 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
@@ -148,7 +148,7 @@ class BinaryDataService(val dataBaseDir: Path,
         convertIfNecessary(
           request.settings.appliedAgglomerate.isDefined && request.dataLayer.category == Category.segmentation && request.cuboid.mag.maxDim <= MaxMagForAgglomerateMapping,
           clippedData,
-          data => agglomerateService.applyAgglomerate(request)(data).toFox,
+          data => agglomerateService.applyAgglomerate(request)(data),
           request
         )
       }.toFox.fillEmpty(Fox.successful(clippedData)) ?~> "Failed to apply agglomerate mapping"
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala
index 36e01971f44..cf8f5c2a135 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala
@@ -111,7 +111,7 @@ class AdHocMeshService(binaryDataService: BinaryDataService,
           Fox.successful(data)
       }
 
-    def applyAgglomerate(data: Array[Byte]): Box[Array[Byte]] =
+    def applyAgglomerate(data: Array[Byte]): Fox[Array[Byte]] =
       request.mapping match {
         case Some(_) =>
           request.mappingType match {
@@ -124,12 +124,12 @@ class AdHocMeshService(binaryDataService: BinaryDataService,
                   DataServiceRequestSettings(halfByte = false, request.mapping, None)
                 )
                 agglomerateService.applyAgglomerate(dataRequest)(data)
-              }.getOrElse(Full(data))
+              }.getOrElse(Fox.successful(data))
             case _ =>
-              Full(data)
+              Fox.successful(data)
           }
         case _ =>
-          Full(data)
+          Fox.successful(data)
       }
 
     def convertData(data: Array[Byte]): Array[T] = {
@@ -193,7 +193,7 @@ class AdHocMeshService(binaryDataService: BinaryDataService,
 
     for {
       data <- binaryDataService.handleDataRequest(dataRequest)
-      agglomerateMappedData <- applyAgglomerate(data).toFox ?~> "failed to apply agglomerate for ad-hoc meshing"
+      agglomerateMappedData <- applyAgglomerate(data) ?~> "failed to apply agglomerate for ad-hoc meshing"
       typedData = convertData(agglomerateMappedData)
       mappedData <- applyMapping(typedData)
       mappedSegmentId <- applyMapping(Array(typedSegmentId)).map(_.head)
diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
index ef176f001ce..ea1aaa70266 100644
--- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
+++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
@@ -3,6 +3,7 @@
 
 # Health endpoint
 GET           /health                                                                                                                                 @com.scalableminds.webknossos.datastore.controllers.Application.health
+GET           /testAgglomerateZarr                                                                                                                    @com.scalableminds.webknossos.datastore.controllers.Application.testAgglomerateZarr
 
 # Read image data
 POST          /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/data                                                              @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaWebknossos(organizationId: String, datasetDirectoryName: String, dataLayerName: String)

From f987ebfafa06487921746768e63349e3725800d9 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 19 May 2025 13:49:32 +0200
Subject: [PATCH 04/22] axisOrder: make y optional

---
 .../datastore/datareaders/AxisOrder.scala     | 22 +++++++++++++------
 .../datastore/datareaders/DatasetHeader.scala | 10 ++++++---
 .../datastore/datareaders/wkw/WKWArray.scala  |  4 ++--
 .../N5CompactMultiscalesExplorer.scala        |  2 +-
 .../datastore/explore/N5Explorer.scala        |  6 ++---
 .../explore/NgffExplorationUtils.scala        | 10 ++++-----
 .../services/AgglomerateService.scala         |  3 ++-
 7 files changed, 35 insertions(+), 22 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala
index 809d4d5e5cf..9d44931e647 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala
@@ -5,14 +5,19 @@ import play.api.libs.json.{JsValue, Json, OFormat}
 
 // Defines the axis order of a DatasetArray. Note that this ignores transpose codecs/ArrayOrder.F/C.
 // Those will have to be applied on individual chunk’s contents.
-case class AxisOrder(x: Int, y: Int, z: Option[Int], c: Option[Int] = None) {
+case class AxisOrder(x: Int, y: Option[Int], z: Option[Int], c: Option[Int] = None) {
 
   def hasZAxis: Boolean = z.isDefined
 
+  def yWithFallback: Int = y match {
+    case Some(value) => value
+    case None        => Math.max(x, c.getOrElse(-1)) + 1
+  }
+
   def zWithFallback: Int = z match {
     case Some(value) => value
     // z is appended to the end of the array (this is reflected in DatasetArray adding 1 at the end of header datasetShape and chunkShape)
-    case None => Math.max(Math.max(x, y), c.getOrElse(-1)) + 1
+    case None => Math.max(Math.max(x, yWithFallback), c.getOrElse(-1)) + 1
   }
 
   def length: Int = {
@@ -27,21 +32,22 @@ object AxisOrder {
   // assumes that the last three elements of the shape are z,y,x (standard in OME NGFF)
   def asZyxFromRank(rank: Int): AxisOrder = AxisOrder.xyz(rank - 1, rank - 2, rank - 3)
 
-  def xyz(x: Int, y: Int, z: Int): AxisOrder = AxisOrder(x, y, Some(z))
+  def xyz(x: Int, y: Int, z: Int): AxisOrder = AxisOrder(x, Some(y), Some(z))
 
-  def xyz: AxisOrder = AxisOrder(0, 1, Some(2))
+  def xyz: AxisOrder = AxisOrder(0, Some(1), Some(2))
 
   // assumes that the last three elements of the shape are (c),x,y,z (which is what WEBKNOSSOS sends to the frontend)
   def asCxyzFromRank(rank: Int): AxisOrder =
     if (rank == 3)
       AxisOrder.xyz(rank - 3, rank - 2, rank - 1)
     else
-      AxisOrder(rank - 3, rank - 2, Some(rank - 1), Some(rank - 4))
+      AxisOrder(rank - 3, Some(rank - 2), Some(rank - 1), Some(rank - 4))
 
   def cxyz: AxisOrder = asCxyzFromRank(rank = 4)
 
   // Additional coordinates are inserted between c and xyz
-  def cAdditionalxyz(rank: Int): AxisOrder = AxisOrder(c = Some(0), x = rank - 3, y = rank - 2, z = Some(rank - 1))
+  def cAdditionalxyz(rank: Int): AxisOrder =
+    AxisOrder(c = Some(0), x = rank - 3, y = Some(rank - 2), z = Some(rank - 1))
   implicit val jsonFormat: OFormat[AxisOrder] = Json.format[AxisOrder]
 }
 
@@ -111,7 +117,9 @@ object FullAxisOrder {
                                      additionalAxes: Option[Seq[AdditionalAxis]]): FullAxisOrder = {
     val asArray: Array[Axis] = Array.fill(rank)(Axis(""))
     asArray(axisOrder.x) = Axis("x")
-    asArray(axisOrder.y) = Axis("y")
+    axisOrder.y.foreach { y =>
+      asArray(y) = Axis("y")
+    }
     axisOrder.c.foreach { c =>
       asArray(c) = Axis("c")
     }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
index 6907d49ecae..095f23a4609 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
@@ -40,13 +40,17 @@ trait DatasetHeader {
 
   def boundingBox(axisOrder: AxisOrder): Option[BoundingBox] =
     datasetShape.flatMap { shape =>
-      if (Math.max(Math.max(axisOrder.x, axisOrder.y), axisOrder.zWithFallback) >= rank && axisOrder.hasZAxis)
+      if (Math.max(Math.max(axisOrder.x, axisOrder.yWithFallback), axisOrder.zWithFallback) >= rank && axisOrder.hasZAxis)
         None
       else {
         if (axisOrder.hasZAxis) {
-          Some(BoundingBox(Vec3Int.zeros, shape(axisOrder.x), shape(axisOrder.y), shape(axisOrder.zWithFallback)))
+          Some(
+            BoundingBox(Vec3Int.zeros,
+                        shape(axisOrder.x),
+                        shape(axisOrder.yWithFallback),
+                        shape(axisOrder.zWithFallback)))
         } else {
-          Some(BoundingBox(Vec3Int.zeros, shape(axisOrder.x), shape(axisOrder.y), 1))
+          Some(BoundingBox(Vec3Int.zeros, shape(axisOrder.x), shape(axisOrder.yWithFallback), 1))
         }
       }
     }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
index f5e7232f9f1..a9dddafdde8 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
@@ -112,7 +112,7 @@ class WKWArray(vaultPath: VaultPath,
 
   private def getChunkIndexInShardIndex(chunkIndex: Array[Int]): Box[Int] = {
     val x = chunkIndex(axisOrder.x)
-    val y = chunkIndex(axisOrder.y)
+    val y = chunkIndex(axisOrder.y.getOrElse(2))
     val z = chunkIndex(axisOrder.z.getOrElse(3))
     val chunkOffsetX = x % header.numChunksPerShardDimension
     val chunkOffsetY = y % header.numChunksPerShardDimension
@@ -122,7 +122,7 @@ class WKWArray(vaultPath: VaultPath,
 
   override protected def getChunkFilename(chunkIndex: Array[Int]): String = {
     val x = chunkIndex(axisOrder.x)
-    val y = chunkIndex(axisOrder.y)
+    val y = chunkIndex(axisOrder.y.getOrElse(2))
     val z = chunkIndex(axisOrder.z.getOrElse(3))
     wkwFilePath(x, y, z)
   }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5CompactMultiscalesExplorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5CompactMultiscalesExplorer.scala
index 6f42482ecac..91c03aff9e4 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5CompactMultiscalesExplorer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5CompactMultiscalesExplorer.scala
@@ -54,7 +54,7 @@ class N5CompactMultiscalesExplorer(implicit val ec: ExecutionContext) extends N5
     for {
       mag <- tryo(
         Vec3Int(downsamplingFactor(axisOrder.x),
-                downsamplingFactor(axisOrder.y),
+                downsamplingFactor(axisOrder.yWithFallback),
                 downsamplingFactor(axisOrder.zWithFallback))).toFox
       magPath = remotePath / s"s$magIndex"
       headerPath = magPath / N5Header.FILENAME_ATTRIBUTES_JSON
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5Explorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5Explorer.scala
index b17f7e1184f..700fde5ca99 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5Explorer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5Explorer.scala
@@ -19,7 +19,7 @@ trait N5Explorer extends RemoteLayerExplorer {
       case Some(units) =>
         for {
           xUnitFactor <- spaceUnitToNmFactor(units(axisOrder.x))
-          yUnitFactor <- spaceUnitToNmFactor(units(axisOrder.y))
+          yUnitFactor <- spaceUnitToNmFactor(units(axisOrder.yWithFallback))
           zUnitFactor <- spaceUnitToNmFactor(units(axisOrder.zWithFallback))
         } yield Vec3Double(xUnitFactor, yUnitFactor, zUnitFactor)
       case None => Fox.successful(Vec3Double(1e3, 1e3, 1e3)) // assume default micrometers
@@ -52,11 +52,11 @@ trait N5Explorer extends RemoteLayerExplorer {
     val cOpt = if (c == -1) None else Some(c)
     for {
       _ <- Fox.fromBool(x >= 0 && y >= 0 && z >= 0) ?~> s"invalid xyz axis order: $x,$y,$z."
-    } yield AxisOrder(x, y, Some(z), cOpt)
+    } yield AxisOrder(x, Some(y), Some(z), cOpt)
   }
 
   protected def extractVoxelSizeInAxisUnits(scale: List[Double], axisOrder: AxisOrder): Fox[Vec3Double] =
-    tryo(Vec3Double(scale(axisOrder.x), scale(axisOrder.y), scale(axisOrder.zWithFallback))).toFox
+    tryo(Vec3Double(scale(axisOrder.x), scale(axisOrder.yWithFallback), scale(axisOrder.zWithFallback))).toFox
 
   protected def layerFromMagsWithAttributes(magsWithAttributes: List[MagWithAttributes],
                                             remotePath: VaultPath): Fox[N5Layer] =
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
index c62c1ddd5b2..948c23f2283 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
@@ -106,9 +106,9 @@ trait NgffExplorationUtils extends FoxImplicits {
       _ <- Fox.fromBool(x >= 0 && y >= 0) ?~> s"invalid xyz axis order: $x,$y,$z. ${x >= 0 && y >= 0}"
     } yield
       if (z >= 0) {
-        AxisOrder(x, y, Some(z), cOpt)
+        AxisOrder(x, Some(y), Some(z), cOpt)
       } else {
-        AxisOrder(x, y, None, cOpt)
+        AxisOrder(x, Some(y), None, cOpt)
       }
   }
 
@@ -116,7 +116,7 @@ trait NgffExplorationUtils extends FoxImplicits {
       implicit ec: ExecutionContext): Fox[LengthUnit] =
     for {
       xUnit <- axes(axisOrder.x).lengthUnit.toFox
-      yUnit <- axes(axisOrder.y).lengthUnit.toFox
+      yUnit <- axes(axisOrder.yWithFallback).lengthUnit.toFox
       zUnitOpt <- Fox.runIf(axisOrder.hasZAxis)(axes(axisOrder.zWithFallback).lengthUnit.toFox)
       units: List[LengthUnit] = List(Some(xUnit), Some(yUnit), zUnitOpt).flatten
     } yield units.minBy(LengthUnit.toNanometer)
@@ -125,7 +125,7 @@ trait NgffExplorationUtils extends FoxImplicits {
       implicit ec: ExecutionContext): Fox[Vec3Double] =
     for {
       xUnitToNm <- axes(axisOrder.x).lengthUnit.map(LengthUnit.toNanometer).toFox
-      yUnitToNm <- axes(axisOrder.y).lengthUnit.map(LengthUnit.toNanometer).toFox
+      yUnitToNm <- axes(axisOrder.yWithFallback).lengthUnit.map(LengthUnit.toNanometer).toFox
       zUnitToNmOpt <- Fox.runIf(axisOrder.hasZAxis)(
         axes(axisOrder.zWithFallback).lengthUnit.map(LengthUnit.toNanometer).toFox)
       xUnitToTarget = xUnitToNm / LengthUnit.toNanometer(unifiedAxisUnit)
@@ -177,7 +177,7 @@ trait NgffExplorationUtils extends FoxImplicits {
     val filtered = coordinateTransforms.filter(_.`type` == "scale")
     val scalesFromTransforms = filtered.flatMap(_.scale)
     val xFactors = scalesFromTransforms.map(_(axisOrder.x))
-    val yFactors = scalesFromTransforms.map(_(axisOrder.y))
+    val yFactors = scalesFromTransforms.map(_(axisOrder.yWithFallback))
     val zFactors = if (axisOrder.hasZAxis) scalesFromTransforms.map(_(axisOrder.zWithFallback)) else Seq(1.0, 1.0)
     Vec3Double(xFactors.product, yFactors.product, zFactors.product)
   }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index e098be3c369..c05f74d4c49 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -10,6 +10,7 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.AgglomerateGraph.{AgglomerateEdge, AgglomerateGraph}
 import com.scalableminds.webknossos.datastore.DataStoreConfig
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, SkeletonTracing, Tree, TreeTypeProto}
+import com.scalableminds.webknossos.datastore.datareaders.AxisOrder
 import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3Array
 import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, SkeletonTracingDefaults}
@@ -64,7 +65,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       zarrArray <- Zarr3Array.open(segmentToAgglomeratePath,
                                    DataSourceId("zarr", "test"),
                                    "layer",
-                                   None,
+                                   Some(AxisOrder(0, None, None)),
                                    None,
                                    None,
                                    sharedChunkContentsCache)(ec, TokenContext(None))

From 7c1cc8bf033606775b79889d59f87adc333371c1 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 22 May 2025 16:08:22 +0200
Subject: [PATCH 05/22] undo attempt to make axisOrder.y optional

---
 .../datastore/datareaders/AxisOrder.scala     | 22 ++++++-------------
 .../datastore/datareaders/DatasetHeader.scala | 10 +++------
 .../datastore/datareaders/wkw/WKWArray.scala  |  4 ++--
 .../N5CompactMultiscalesExplorer.scala        |  2 +-
 .../datastore/explore/N5Explorer.scala        |  6 ++---
 .../explore/NgffExplorationUtils.scala        | 10 ++++-----
 6 files changed, 21 insertions(+), 33 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala
index 9d44931e647..809d4d5e5cf 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala
@@ -5,19 +5,14 @@ import play.api.libs.json.{JsValue, Json, OFormat}
 
 // Defines the axis order of a DatasetArray. Note that this ignores transpose codecs/ArrayOrder.F/C.
 // Those will have to be applied on individual chunk’s contents.
-case class AxisOrder(x: Int, y: Option[Int], z: Option[Int], c: Option[Int] = None) {
+case class AxisOrder(x: Int, y: Int, z: Option[Int], c: Option[Int] = None) {
 
   def hasZAxis: Boolean = z.isDefined
 
-  def yWithFallback: Int = y match {
-    case Some(value) => value
-    case None        => Math.max(x, c.getOrElse(-1)) + 1
-  }
-
   def zWithFallback: Int = z match {
     case Some(value) => value
     // z is appended to the end of the array (this is reflected in DatasetArray adding 1 at the end of header datasetShape and chunkShape)
-    case None => Math.max(Math.max(x, yWithFallback), c.getOrElse(-1)) + 1
+    case None => Math.max(Math.max(x, y), c.getOrElse(-1)) + 1
   }
 
   def length: Int = {
@@ -32,22 +27,21 @@ object AxisOrder {
   // assumes that the last three elements of the shape are z,y,x (standard in OME NGFF)
   def asZyxFromRank(rank: Int): AxisOrder = AxisOrder.xyz(rank - 1, rank - 2, rank - 3)
 
-  def xyz(x: Int, y: Int, z: Int): AxisOrder = AxisOrder(x, Some(y), Some(z))
+  def xyz(x: Int, y: Int, z: Int): AxisOrder = AxisOrder(x, y, Some(z))
 
-  def xyz: AxisOrder = AxisOrder(0, Some(1), Some(2))
+  def xyz: AxisOrder = AxisOrder(0, 1, Some(2))
 
   // assumes that the last three elements of the shape are (c),x,y,z (which is what WEBKNOSSOS sends to the frontend)
   def asCxyzFromRank(rank: Int): AxisOrder =
     if (rank == 3)
       AxisOrder.xyz(rank - 3, rank - 2, rank - 1)
     else
-      AxisOrder(rank - 3, Some(rank - 2), Some(rank - 1), Some(rank - 4))
+      AxisOrder(rank - 3, rank - 2, Some(rank - 1), Some(rank - 4))
 
   def cxyz: AxisOrder = asCxyzFromRank(rank = 4)
 
   // Additional coordinates are inserted between c and xyz
-  def cAdditionalxyz(rank: Int): AxisOrder =
-    AxisOrder(c = Some(0), x = rank - 3, y = Some(rank - 2), z = Some(rank - 1))
+  def cAdditionalxyz(rank: Int): AxisOrder = AxisOrder(c = Some(0), x = rank - 3, y = rank - 2, z = Some(rank - 1))
   implicit val jsonFormat: OFormat[AxisOrder] = Json.format[AxisOrder]
 }
 
@@ -117,9 +111,7 @@ object FullAxisOrder {
                                      additionalAxes: Option[Seq[AdditionalAxis]]): FullAxisOrder = {
     val asArray: Array[Axis] = Array.fill(rank)(Axis(""))
     asArray(axisOrder.x) = Axis("x")
-    axisOrder.y.foreach { y =>
-      asArray(y) = Axis("y")
-    }
+    asArray(axisOrder.y) = Axis("y")
     axisOrder.c.foreach { c =>
       asArray(c) = Axis("c")
     }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
index 095f23a4609..6907d49ecae 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
@@ -40,17 +40,13 @@ trait DatasetHeader {
 
   def boundingBox(axisOrder: AxisOrder): Option[BoundingBox] =
     datasetShape.flatMap { shape =>
-      if (Math.max(Math.max(axisOrder.x, axisOrder.yWithFallback), axisOrder.zWithFallback) >= rank && axisOrder.hasZAxis)
+      if (Math.max(Math.max(axisOrder.x, axisOrder.y), axisOrder.zWithFallback) >= rank && axisOrder.hasZAxis)
         None
       else {
         if (axisOrder.hasZAxis) {
-          Some(
-            BoundingBox(Vec3Int.zeros,
-                        shape(axisOrder.x),
-                        shape(axisOrder.yWithFallback),
-                        shape(axisOrder.zWithFallback)))
+          Some(BoundingBox(Vec3Int.zeros, shape(axisOrder.x), shape(axisOrder.y), shape(axisOrder.zWithFallback)))
         } else {
-          Some(BoundingBox(Vec3Int.zeros, shape(axisOrder.x), shape(axisOrder.yWithFallback), 1))
+          Some(BoundingBox(Vec3Int.zeros, shape(axisOrder.x), shape(axisOrder.y), 1))
         }
       }
     }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
index a9dddafdde8..f5e7232f9f1 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
@@ -112,7 +112,7 @@ class WKWArray(vaultPath: VaultPath,
 
   private def getChunkIndexInShardIndex(chunkIndex: Array[Int]): Box[Int] = {
     val x = chunkIndex(axisOrder.x)
-    val y = chunkIndex(axisOrder.y.getOrElse(2))
+    val y = chunkIndex(axisOrder.y)
     val z = chunkIndex(axisOrder.z.getOrElse(3))
     val chunkOffsetX = x % header.numChunksPerShardDimension
     val chunkOffsetY = y % header.numChunksPerShardDimension
@@ -122,7 +122,7 @@ class WKWArray(vaultPath: VaultPath,
 
   override protected def getChunkFilename(chunkIndex: Array[Int]): String = {
     val x = chunkIndex(axisOrder.x)
-    val y = chunkIndex(axisOrder.y.getOrElse(2))
+    val y = chunkIndex(axisOrder.y)
     val z = chunkIndex(axisOrder.z.getOrElse(3))
     wkwFilePath(x, y, z)
   }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5CompactMultiscalesExplorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5CompactMultiscalesExplorer.scala
index 91c03aff9e4..6f42482ecac 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5CompactMultiscalesExplorer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5CompactMultiscalesExplorer.scala
@@ -54,7 +54,7 @@ class N5CompactMultiscalesExplorer(implicit val ec: ExecutionContext) extends N5
     for {
       mag <- tryo(
         Vec3Int(downsamplingFactor(axisOrder.x),
-                downsamplingFactor(axisOrder.yWithFallback),
+                downsamplingFactor(axisOrder.y),
                 downsamplingFactor(axisOrder.zWithFallback))).toFox
       magPath = remotePath / s"s$magIndex"
       headerPath = magPath / N5Header.FILENAME_ATTRIBUTES_JSON
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5Explorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5Explorer.scala
index 700fde5ca99..b17f7e1184f 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5Explorer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/N5Explorer.scala
@@ -19,7 +19,7 @@ trait N5Explorer extends RemoteLayerExplorer {
       case Some(units) =>
         for {
           xUnitFactor <- spaceUnitToNmFactor(units(axisOrder.x))
-          yUnitFactor <- spaceUnitToNmFactor(units(axisOrder.yWithFallback))
+          yUnitFactor <- spaceUnitToNmFactor(units(axisOrder.y))
           zUnitFactor <- spaceUnitToNmFactor(units(axisOrder.zWithFallback))
         } yield Vec3Double(xUnitFactor, yUnitFactor, zUnitFactor)
       case None => Fox.successful(Vec3Double(1e3, 1e3, 1e3)) // assume default micrometers
@@ -52,11 +52,11 @@ trait N5Explorer extends RemoteLayerExplorer {
     val cOpt = if (c == -1) None else Some(c)
     for {
       _ <- Fox.fromBool(x >= 0 && y >= 0 && z >= 0) ?~> s"invalid xyz axis order: $x,$y,$z."
-    } yield AxisOrder(x, Some(y), Some(z), cOpt)
+    } yield AxisOrder(x, y, Some(z), cOpt)
   }
 
   protected def extractVoxelSizeInAxisUnits(scale: List[Double], axisOrder: AxisOrder): Fox[Vec3Double] =
-    tryo(Vec3Double(scale(axisOrder.x), scale(axisOrder.yWithFallback), scale(axisOrder.zWithFallback))).toFox
+    tryo(Vec3Double(scale(axisOrder.x), scale(axisOrder.y), scale(axisOrder.zWithFallback))).toFox
 
   protected def layerFromMagsWithAttributes(magsWithAttributes: List[MagWithAttributes],
                                             remotePath: VaultPath): Fox[N5Layer] =
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
index 948c23f2283..c62c1ddd5b2 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
@@ -106,9 +106,9 @@ trait NgffExplorationUtils extends FoxImplicits {
       _ <- Fox.fromBool(x >= 0 && y >= 0) ?~> s"invalid xyz axis order: $x,$y,$z. ${x >= 0 && y >= 0}"
     } yield
       if (z >= 0) {
-        AxisOrder(x, Some(y), Some(z), cOpt)
+        AxisOrder(x, y, Some(z), cOpt)
       } else {
-        AxisOrder(x, Some(y), None, cOpt)
+        AxisOrder(x, y, None, cOpt)
       }
   }
 
@@ -116,7 +116,7 @@ trait NgffExplorationUtils extends FoxImplicits {
       implicit ec: ExecutionContext): Fox[LengthUnit] =
     for {
       xUnit <- axes(axisOrder.x).lengthUnit.toFox
-      yUnit <- axes(axisOrder.yWithFallback).lengthUnit.toFox
+      yUnit <- axes(axisOrder.y).lengthUnit.toFox
       zUnitOpt <- Fox.runIf(axisOrder.hasZAxis)(axes(axisOrder.zWithFallback).lengthUnit.toFox)
       units: List[LengthUnit] = List(Some(xUnit), Some(yUnit), zUnitOpt).flatten
     } yield units.minBy(LengthUnit.toNanometer)
@@ -125,7 +125,7 @@ trait NgffExplorationUtils extends FoxImplicits {
       implicit ec: ExecutionContext): Fox[Vec3Double] =
     for {
       xUnitToNm <- axes(axisOrder.x).lengthUnit.map(LengthUnit.toNanometer).toFox
-      yUnitToNm <- axes(axisOrder.yWithFallback).lengthUnit.map(LengthUnit.toNanometer).toFox
+      yUnitToNm <- axes(axisOrder.y).lengthUnit.map(LengthUnit.toNanometer).toFox
       zUnitToNmOpt <- Fox.runIf(axisOrder.hasZAxis)(
         axes(axisOrder.zWithFallback).lengthUnit.map(LengthUnit.toNanometer).toFox)
       xUnitToTarget = xUnitToNm / LengthUnit.toNanometer(unifiedAxisUnit)
@@ -177,7 +177,7 @@ trait NgffExplorationUtils extends FoxImplicits {
     val filtered = coordinateTransforms.filter(_.`type` == "scale")
     val scalesFromTransforms = filtered.flatMap(_.scale)
     val xFactors = scalesFromTransforms.map(_(axisOrder.x))
-    val yFactors = scalesFromTransforms.map(_(axisOrder.yWithFallback))
+    val yFactors = scalesFromTransforms.map(_(axisOrder.y))
     val zFactors = if (axisOrder.hasZAxis) scalesFromTransforms.map(_(axisOrder.zWithFallback)) else Seq(1.0, 1.0)
     Vec3Double(xFactors.product, yFactors.product, zFactors.product)
   }

From 5af14fcbb83f78971eeea7c86c89353f96b3a16c Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 22 May 2025 16:33:35 +0200
Subject: [PATCH 06/22] read multi array, ignoring underlying storage and axis
 order

---
 .../datastore/controllers/Application.scala   |  2 +-
 .../datastore/datareaders/DatasetArray.scala  | 63 ++++++++++++++++---
 .../services/AgglomerateService.scala         | 10 +--
 3 files changed, 62 insertions(+), 13 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
index 3ede25cb42f..f22b572a55d 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
@@ -44,7 +44,7 @@ class Application @Inject()(redisClient: DataStoreRedisStore,
     log() {
       for {
         data <- agglomerateService.readFromSegmentToAgglomerate
-      } yield Ok(s"got ${data.length} bytes")
+      } yield Ok(s"got ${data.getSize} elements of type ${data.getDataType}: ${data.toString}")
     }
   }
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
index 2a00dd70a7b..242fc71453a 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
@@ -116,9 +116,8 @@ class DatasetArray(vaultPath: VaultPath,
   }
 
   // returns byte array in fortran-order with little-endian values
-  // TODO should possibly be private again
-  def readBytes(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext,
-                                                       tc: TokenContext): Fox[Array[Byte]] =
+  private def readBytes(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext,
+                                                               tc: TokenContext): Fox[Array[Byte]] =
     for {
       typedMultiArray <- readAsFortranOrder(shape, offset)
       asBytes <- BytesConverter.toByteArray(typedMultiArray, header.resolvedDataType, ByteOrder.LITTLE_ENDIAN).toFox
@@ -158,7 +157,7 @@ class DatasetArray(vaultPath: VaultPath,
                                                       fullAxisOrder.permuteIndicesArrayToWk(chunkShape),
                                                       shape,
                                                       totalOffset)
-    if (partialCopyingIsNotNeeded(shape, totalOffset, chunkIndices)) {
+    if (partialCopyingIsNotNeededForWkOrder(shape, totalOffset, chunkIndices)) {
       for {
         chunkIndex <- chunkIndices.headOption.toFox
         sourceChunk: MultiArray <- getSourceChunkDataWithCache(fullAxisOrder.permuteIndicesWkToArray(chunkIndex),
@@ -185,10 +184,44 @@ class DatasetArray(vaultPath: VaultPath,
     }
   }
 
+  def readAsMultiArray(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext,
+                                                              tc: TokenContext): Fox[MultiArray] = {
+    val totalOffset: Array[Int] = offset.zip(header.voxelOffset).map { case (o, v) => o - v }.padTo(offset.length, 0)
+    val chunkIndices = ChunkUtils.computeChunkIndices(datasetShape, chunkShape, shape, totalOffset)
+    if (partialCopyingIsNotNeededForMultiArray(shape, totalOffset, chunkIndices)) {
+      for {
+        chunkIndex <- chunkIndices.headOption.toFox
+        sourceChunk: MultiArray <- getSourceChunkDataWithCache(chunkIndex, useSkipTypingShortcut = true)
+      } yield sourceChunk
+    } else {
+      val targetBuffer = MultiArrayUtils.createDataBuffer(header.resolvedDataType, shape)
+      val targetMultiArray = MultiArrayUtils.createArrayWithGivenStorage(targetBuffer, shape.reverse)
+      val copiedFuture = Fox.combined(chunkIndices.map { chunkIndex: Array[Int] =>
+        for {
+          sourceChunk: MultiArray <- getSourceChunkDataWithCache(chunkIndex)
+          offsetInChunk = computeOffsetInChunkIgnoringAxisOrder(chunkIndex, totalOffset).reverse
+          _ <- tryo(MultiArrayUtils.copyRange(offsetInChunk, sourceChunk, targetMultiArray)).toFox ?~> formatCopyRangeErrorWithoutAxisOrder(
+            offsetInChunk,
+            sourceChunk,
+            targetMultiArray)
+        } yield ()
+      })
+      for {
+        _ <- copiedFuture
+      } yield targetMultiArray
+    }
+  }
+
   private def formatCopyRangeError(offsetInChunk: Array[Int], sourceChunk: MultiArray, target: MultiArray): String =
     s"Copying data from dataset chunk failed. Chunk shape (F): ${printAsOuterF(sourceChunk.getShape)}, target shape (F): ${printAsOuterF(
       target.getShape)}, offsetInChunk: ${printAsOuterF(offsetInChunk)}. Axis order (C): $fullAxisOrder (outer: ${fullAxisOrder.toStringWk})"
 
+  private def formatCopyRangeErrorWithoutAxisOrder(offsetInChunk: Array[Int],
+                                                   sourceChunk: MultiArray,
+                                                   target: MultiArray): String =
+    s"Copying data from dataset chunk failed. Chunk shape ${sourceChunk.getShape.mkString(",")}, target shape ${target.getShape
+      .mkString(",")}, offsetInChunk: ${offsetInChunk.mkString(",")}"
+
   protected def getShardedChunkPathAndRange(
       chunkIndex: Array[Int])(implicit ec: ExecutionContext, tc: TokenContext): Fox[(VaultPath, NumericRange[Long])] =
     ??? // Defined in subclass
@@ -225,9 +258,20 @@ class DatasetArray(vaultPath: VaultPath,
       chunkIndex.drop(1).mkString(header.dimension_separator.toString) // (c),x,y,z -> z is dropped in 2d case
     }
 
-  private def partialCopyingIsNotNeeded(bufferShape: Array[Int],
-                                        globalOffset: Array[Int],
-                                        chunkIndices: List[Array[Int]]): Boolean =
+  private def partialCopyingIsNotNeededForMultiArray(bufferShape: Array[Int],
+                                                     globalOffset: Array[Int],
+                                                     chunkIndices: List[Array[Int]]): Boolean =
+    chunkIndices match {
+      case chunkIndex :: Nil =>
+        val offsetInChunk = computeOffsetInChunkIgnoringAxisOrder(chunkIndex, globalOffset)
+        isZeroOffset(offsetInChunk) &&
+        isBufferShapeEqualChunkShape(bufferShape)
+      case _ => false
+    }
+
+  private def partialCopyingIsNotNeededForWkOrder(bufferShape: Array[Int],
+                                                  globalOffset: Array[Int],
+                                                  chunkIndices: List[Array[Int]]): Boolean =
     chunkIndices match {
       case chunkIndex :: Nil =>
         val offsetInChunk = computeOffsetInChunk(chunkIndex, globalOffset)
@@ -249,6 +293,11 @@ class DatasetArray(vaultPath: VaultPath,
       globalOffset(dim) - (chunkIndex(dim) * fullAxisOrder.permuteIndicesArrayToWk(chunkShape)(dim))
     }.toArray
 
+  private def computeOffsetInChunkIgnoringAxisOrder(chunkIndex: Array[Int], globalOffset: Array[Int]): Array[Int] =
+    chunkIndex.indices.map { dim =>
+      globalOffset(dim) - (chunkIndex(dim) * chunkShape(dim))
+    }.toArray
+
   override def toString: String =
     s"${getClass.getCanonicalName} fullAxisOrder=$fullAxisOrder shape=${header.datasetShape.map(s => printAsInner(s))} chunkShape=${printAsInner(
       header.chunkShape)} dtype=${header.resolvedDataType} fillValue=${header.fillValueNumber}, ${header.compressorImpl}, byteOrder=${header.byteOrder}, vault=${vaultPath.summary}}"
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index c05f74d4c49..a50489364b4 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -54,10 +54,10 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
     AlfuCache(maxSizeKiloBytes, weighFn = Some(cacheWeight))
   }
 
-  def readFromSegmentToAgglomerate(implicit ec: ExecutionContext): Fox[Array[Byte]] = {
+  def readFromSegmentToAgglomerate(implicit ec: ExecutionContext): Fox[ucar.ma2.Array] = {
     val zarrGroupPath =
       dataBaseDir
-        .resolve("sample_organization/test-agglomerate-file-zarr/segmentation/agglomerates/agglomerate_view_5")
+        .resolve("sample_organization/test-agglomerate-file-zarr/segmentation/agglomerates/agglomerate_view_55")
         .toAbsolutePath
     for {
       groupVaultPath <- dataVaultService.getVaultPath(RemoteSourceDescriptor(new URI(s"file://$zarrGroupPath"), None))
@@ -65,12 +65,12 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       zarrArray <- Zarr3Array.open(segmentToAgglomeratePath,
                                    DataSourceId("zarr", "test"),
                                    "layer",
-                                   Some(AxisOrder(0, None, None)),
+                                   None,
                                    None,
                                    None,
                                    sharedChunkContentsCache)(ec, TokenContext(None))
-      read <- zarrArray.readBytes(Array(5), Array(0))(ec, TokenContext(None))
-      _ = logger.info(s"read ${read.length} bytes from agglomerate file")
+      read <- zarrArray.readAsMultiArray(Array(10), Array(2))(ec, TokenContext(None))
+      _ = logger.info(s"read ${read.getSize} bytes from agglomerate file")
     } yield read
   }
 

From f055c1e6970f41d273e5fec736c88d7161390ada Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 27 May 2025 10:07:00 +0200
Subject: [PATCH 07/22] apply agglomerate

---
 .../datastore/datareaders/DatasetArray.scala  |  5 +-
 .../datareaders/zarr3/Zarr3Array.scala        |  2 +-
 .../services/AgglomerateService.scala         | 76 +++++++++++++++----
 3 files changed, 68 insertions(+), 15 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
index 242fc71453a..d1080732ec5 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
@@ -8,6 +8,7 @@ import com.scalableminds.webknossos.datastore.datavault.VaultPath
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate
 import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis
+import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.Box.tryo
 import ucar.ma2.{Array => MultiArray}
 
@@ -26,7 +27,8 @@ class DatasetArray(vaultPath: VaultPath,
                    channelIndex: Option[Int],
                    additionalAxes: Option[Seq[AdditionalAxis]],
                    sharedChunkContentsCache: AlfuCache[String, MultiArray])
-    extends FoxImplicits {
+    extends FoxImplicits
+    with LazyLogging {
 
   protected lazy val fullAxisOrder: FullAxisOrder =
     FullAxisOrder.fromAxisOrderAndAdditionalAxes(rank, axisOrder, additionalAxes)
@@ -242,6 +244,7 @@ class DatasetArray(vaultPath: VaultPath,
     if (header.isSharded) {
       for {
         (shardPath, chunkRange) <- getShardedChunkPathAndRange(chunkIndex) ?~> "chunk.getShardedPathAndRange.failed"
+        _ = logger.info(s"chunk cache miss for $shardPath chunk ${chunkIndex.mkString(",")} ")
         chunkShape = chunkShapeAtIndex(chunkIndex)
         multiArray <- chunkReader.read(shardPath, chunkShape, Some(chunkRange), useSkipTypingShortcut)
       } yield multiArray
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3Array.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3Array.scala
index 5a7d0c7b807..ce417854119 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3Array.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3Array.scala
@@ -121,7 +121,7 @@ class Zarr3Array(vaultPath: VaultPath,
   private def readAndParseShardIndex(shardPath: VaultPath)(implicit ec: ExecutionContext,
                                                            tc: TokenContext): Fox[Array[(Long, Long)]] =
     for {
-      shardIndexRaw <- readShardIndex(shardPath)
+      shardIndexRaw <- readShardIndex(shardPath) ?~> "readShardIndex.failed"
       parsed = parseShardIndex(shardIndexRaw)
     } yield parsed
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index a50489364b4..a6a97c0aad0 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -10,7 +10,7 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.AgglomerateGraph.{AgglomerateEdge, AgglomerateGraph}
 import com.scalableminds.webknossos.datastore.DataStoreConfig
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, SkeletonTracing, Tree, TreeTypeProto}
-import com.scalableminds.webknossos.datastore.datareaders.AxisOrder
+import com.scalableminds.webknossos.datastore.datareaders.{AxisOrder, DatasetArray}
 import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3Array
 import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, SkeletonTracingDefaults}
@@ -39,6 +39,9 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
   private val agglomerateDir = "agglomerates"
   private val agglomerateFileExtension = ""
 
+  private lazy val openArraysCache = AlfuCache[String, DatasetArray]()
+
+  // TODO unify with existing chunkContentsCache from binaryDataService
   private lazy val sharedChunkContentsCache: AlfuCache[String, MultiArray] = {
     // Used by DatasetArray-based datasets. Measure item weight in kilobytes because the weigher can only return int, not long
 
@@ -54,14 +57,32 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
     AlfuCache(maxSizeKiloBytes, weighFn = Some(cacheWeight))
   }
 
-  def readFromSegmentToAgglomerate(implicit ec: ExecutionContext): Fox[ucar.ma2.Array] = {
+  def readFromSegmentToAgglomerate(implicit ec: ExecutionContext, tc: TokenContext): Fox[ucar.ma2.Array] =
+    for {
+      zarrArray <- openZarrArrayCached("segment_to_agglomerate")
+      read <- zarrArray.readAsMultiArray(Array(10), Array(2))
+      _ = logger.info(s"read ${read.getSize} elements from agglomerate file segmentToAgglomerate")
+    } yield read
+
+  private def mapSingleSegment(segmentId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Long] =
+    for {
+      zarrArray <- openZarrArrayCached("segment_to_agglomerate")
+      // TODO remove the toInt
+      asMultiArray <- zarrArray.readAsMultiArray(offset = Array(segmentId.toInt), shape = Array(1))
+    } yield asMultiArray.getLong(0)
+
+  private def openZarrArrayCached(zarrArrayName: String)(implicit ec: ExecutionContext, tc: TokenContext) =
+    openArraysCache.getOrLoad(zarrArrayName, zarrArrayName => openZarrArray(zarrArrayName))
+
+  private def openZarrArray(zarrArrayName: String)(implicit ec: ExecutionContext,
+                                                   tc: TokenContext): Fox[DatasetArray] = {
     val zarrGroupPath =
       dataBaseDir
         .resolve("sample_organization/test-agglomerate-file-zarr/segmentation/agglomerates/agglomerate_view_55")
         .toAbsolutePath
     for {
       groupVaultPath <- dataVaultService.getVaultPath(RemoteSourceDescriptor(new URI(s"file://$zarrGroupPath"), None))
-      segmentToAgglomeratePath = groupVaultPath / "segment_to_agglomerate"
+      segmentToAgglomeratePath = groupVaultPath / zarrArrayName
       zarrArray <- Zarr3Array.open(segmentToAgglomeratePath,
                                    DataSourceId("zarr", "test"),
                                    "layer",
@@ -69,21 +90,50 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
                                    None,
                                    None,
                                    sharedChunkContentsCache)(ec, TokenContext(None))
-      read <- zarrArray.readAsMultiArray(Array(10), Array(2))(ec, TokenContext(None))
-      _ = logger.info(s"read ${read.getSize} bytes from agglomerate file")
-    } yield read
+    } yield zarrArray
   }
 
-  def applyAgglomerateHdf5(request: DataServiceDataRequest)(data: Array[Byte])(
-      implicit ec: ExecutionContext): Fox[Array[Byte]] = {
+  def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(implicit ec: ExecutionContext,
+                                                                           tc: TokenContext): Fox[Array[Byte]] = {
 
     val agglomerateFileKey = AgglomerateFileKey.fromDataRequest(request)
-
     val zarrGroupPath = agglomerateFileKey.zarrGroupPath(dataBaseDir, agglomerateDir).toAbsolutePath
 
-    for {
-      _ <- readFromSegmentToAgglomerate
-    } yield data
+    def convertToAgglomerate(segmentIds: Array[Long],
+                             bytesPerElement: Int,
+                             putToBufferFunction: (ByteBuffer, Long) => ByteBuffer): Fox[Array[Byte]] =
+      for {
+        agglomerateIds <- Fox.serialCombined(segmentIds)(mapSingleSegment)
+        mappedBytes = agglomerateIds
+          .foldLeft(ByteBuffer.allocate(bytesPerElement * segmentIds.length).order(ByteOrder.LITTLE_ENDIAN))(
+            putToBufferFunction)
+          .array
+      } yield mappedBytes
+
+    val bytesPerElement = ElementClass.bytesPerElement(request.dataLayer.elementClass)
+    /* Every value of the segmentation data needs to be converted to Long to then look up the
+       agglomerate id in the segment-to-agglomerate array.
+       The value is first converted to the primitive signed number types, and then converted
+       to Long via uByteToLong, uShortToLong etc, which perform bitwise and to take care of
+       the unsigned semantics. Using functions avoids allocating intermediate SegmentInteger objects.
+       Allocating a fixed-length LongBuffer first is a further performance optimization.
+     */
+    convertData(data, request.dataLayer.elementClass) match {
+      case data: Array[Byte] =>
+        val longBuffer = LongBuffer.allocate(data.length)
+        data.foreach(e => longBuffer.put(uByteToLong(e)))
+        convertToAgglomerate(longBuffer.array, bytesPerElement, putByte)
+      case data: Array[Short] =>
+        val longBuffer = LongBuffer.allocate(data.length)
+        data.foreach(e => longBuffer.put(uShortToLong(e)))
+        convertToAgglomerate(longBuffer.array, bytesPerElement, putShort)
+      case data: Array[Int] =>
+        val longBuffer = LongBuffer.allocate(data.length)
+        data.foreach(e => longBuffer.put(uIntToLong(e)))
+        convertToAgglomerate(longBuffer.array, bytesPerElement, putInt)
+      case data: Array[Long] => convertToAgglomerate(data, bytesPerElement, putLong)
+      case _                 => Fox.successful(data)
+    }
 
   }
 
@@ -122,7 +172,7 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
   def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(
       implicit ec: ExecutionContext): Fox[Array[Byte]] =
     if (true) {
-      zarrAgglomerateService.applyAgglomerateHdf5(request)(data)
+      zarrAgglomerateService.applyAgglomerate(request)(data)(ec, TokenContext(None))
     } else applyAgglomerateHdf5(request)(data).toFox
 
   private def applyAgglomerateHdf5(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] = tryo {

From 13ff0e3ca4588119940fcc507c9b955873d6ca73 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 27 May 2025 10:13:28 +0200
Subject: [PATCH 08/22] offset can be long; pass tokencontext

---
 .../datastore/datareaders/ChunkUtils.scala         |  6 +++---
 .../datastore/datareaders/DatasetArray.scala       | 14 +++++++-------
 .../datastore/services/AgglomerateService.scala    |  9 ++++-----
 .../datastore/services/BinaryDataService.scala     |  3 ++-
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala
index 60378c05c7b..f2917923e4a 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala
@@ -4,7 +4,7 @@ object ChunkUtils {
   def computeChunkIndices(arrayShapeOpt: Option[Array[Int]],
                           arrayChunkShape: Array[Int],
                           selectedShape: Array[Int],
-                          selectedOffset: Array[Int]): List[Array[Int]] = {
+                          selectedOffset: Array[Long]): List[Array[Int]] = {
     val nDims = arrayChunkShape.length
     val start = new Array[Int](nDims)
     val end = new Array[Int](nDims)
@@ -12,10 +12,10 @@ object ChunkUtils {
     for (dim <- 0 until nDims) {
       val largestPossibleIndex = arrayShapeOpt.map(arrayShape => (arrayShape(dim) - 1) / arrayChunkShape(dim))
       val smallestPossibleIndex = 0
-      val startIndexRaw = selectedOffset(dim) / arrayChunkShape(dim)
+      val startIndexRaw = (selectedOffset(dim) / arrayChunkShape(dim)).toInt
       val startIndexClamped =
         Math.max(smallestPossibleIndex, Math.min(largestPossibleIndex.getOrElse(startIndexRaw), startIndexRaw))
-      val endIndexRaw = (selectedOffset(dim) + selectedShape(dim) - 1) / arrayChunkShape(dim)
+      val endIndexRaw = ((selectedOffset(dim) + selectedShape(dim) - 1) / arrayChunkShape(dim)).toInt
       val endIndexClampedToBbox =
         Math.max(smallestPossibleIndex, Math.min(largestPossibleIndex.getOrElse(endIndexRaw), endIndexRaw))
       val endIndexClamped = Math.max(startIndexClamped, endIndexClampedToBbox) // end index must be greater or equal to start index
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
index d1080732ec5..76ab19b7e84 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
@@ -158,7 +158,7 @@ class DatasetArray(vaultPath: VaultPath,
     val chunkIndices = ChunkUtils.computeChunkIndices(datasetShape.map(fullAxisOrder.permuteIndicesArrayToWk),
                                                       fullAxisOrder.permuteIndicesArrayToWk(chunkShape),
                                                       shape,
-                                                      totalOffset)
+                                                      totalOffset.map(_.toLong))
     if (partialCopyingIsNotNeededForWkOrder(shape, totalOffset, chunkIndices)) {
       for {
         chunkIndex <- chunkIndices.headOption.toFox
@@ -186,9 +186,9 @@ class DatasetArray(vaultPath: VaultPath,
     }
   }
 
-  def readAsMultiArray(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext,
-                                                              tc: TokenContext): Fox[MultiArray] = {
-    val totalOffset: Array[Int] = offset.zip(header.voxelOffset).map { case (o, v) => o - v }.padTo(offset.length, 0)
+  def readAsMultiArray(shape: Array[Int], offset: Array[Long])(implicit ec: ExecutionContext,
+                                                               tc: TokenContext): Fox[MultiArray] = {
+    val totalOffset: Array[Long] = offset.zip(header.voxelOffset).map { case (o, v) => o - v }.padTo(offset.length, 0)
     val chunkIndices = ChunkUtils.computeChunkIndices(datasetShape, chunkShape, shape, totalOffset)
     if (partialCopyingIsNotNeededForMultiArray(shape, totalOffset, chunkIndices)) {
       for {
@@ -262,7 +262,7 @@ class DatasetArray(vaultPath: VaultPath,
     }
 
   private def partialCopyingIsNotNeededForMultiArray(bufferShape: Array[Int],
-                                                     globalOffset: Array[Int],
+                                                     globalOffset: Array[Long],
                                                      chunkIndices: List[Array[Int]]): Boolean =
     chunkIndices match {
       case chunkIndex :: Nil =>
@@ -296,9 +296,9 @@ class DatasetArray(vaultPath: VaultPath,
       globalOffset(dim) - (chunkIndex(dim) * fullAxisOrder.permuteIndicesArrayToWk(chunkShape)(dim))
     }.toArray
 
-  private def computeOffsetInChunkIgnoringAxisOrder(chunkIndex: Array[Int], globalOffset: Array[Int]): Array[Int] =
+  private def computeOffsetInChunkIgnoringAxisOrder(chunkIndex: Array[Int], globalOffset: Array[Long]): Array[Int] =
     chunkIndex.indices.map { dim =>
-      globalOffset(dim) - (chunkIndex(dim) * chunkShape(dim))
+      (globalOffset(dim) - (chunkIndex(dim).toLong * chunkShape(dim).toLong)).toInt
     }.toArray
 
   override def toString: String =
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index a6a97c0aad0..db34ed3c269 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -67,8 +67,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
   private def mapSingleSegment(segmentId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Long] =
     for {
       zarrArray <- openZarrArrayCached("segment_to_agglomerate")
-      // TODO remove the toInt
-      asMultiArray <- zarrArray.readAsMultiArray(offset = Array(segmentId.toInt), shape = Array(1))
+      asMultiArray <- zarrArray.readAsMultiArray(offset = Array(segmentId), shape = Array(1))
     } yield asMultiArray.getLong(0)
 
   private def openZarrArrayCached(zarrArrayName: String)(implicit ec: ExecutionContext, tc: TokenContext) =
@@ -169,10 +168,10 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
       .toSet ++ Set("agglomerate_view_5") // TODO
   }
 
-  def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(
-      implicit ec: ExecutionContext): Fox[Array[Byte]] =
+  def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(implicit ec: ExecutionContext,
+                                                                           tc: TokenContext): Fox[Array[Byte]] =
     if (true) {
-      zarrAgglomerateService.applyAgglomerate(request)(data)(ec, TokenContext(None))
+      zarrAgglomerateService.applyAgglomerate(request)(data)
     } else applyAgglomerateHdf5(request)(data).toFox
 
   private def applyAgglomerateHdf5(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] = tryo {
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
index 6d82ad62077..58d4dd006e1 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
@@ -136,7 +136,8 @@ class BinaryDataService(val dataBaseDir: Path,
     Full(outputArray)
   }
 
-  private def convertAccordingToRequest(request: DataServiceDataRequest, inputArray: Array[Byte]): Fox[Array[Byte]] =
+  private def convertAccordingToRequest(request: DataServiceDataRequest, inputArray: Array[Byte])(
+      implicit tc: TokenContext): Fox[Array[Byte]] =
     for {
       clippedData <- convertIfNecessary(
         !request.cuboid.toMag1BoundingBox.isFullyContainedIn(request.dataLayer.boundingBox),

From d8533899ed1abe0a850c6ba48faae305603f69a6 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 27 May 2025 11:54:00 +0200
Subject: [PATCH 09/22] WIP read agglomerate skeleton

---
 conf/application.conf                         |   2 +-
 .../controllers/DataSourceController.scala    |   8 +-
 .../datastore/datareaders/DatasetArray.scala  |   4 +-
 .../services/AgglomerateService.scala         | 328 +++++++++++-------
 4 files changed, 218 insertions(+), 124 deletions(-)

diff --git a/conf/application.conf b/conf/application.conf
index aac6419d24e..c295317e578 100644
--- a/conf/application.conf
+++ b/conf/application.conf
@@ -347,4 +347,4 @@ pidfile.path = "/dev/null"
 
 
 # uncomment these lines for faster restart during local backend development (but beware the then-missing features):
-#slick.checkSchemaOnStartup = false
+slick.checkSchemaOnStartup = false
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
index c6f84eaf0e0..c7bcdb229de 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
@@ -277,9 +277,11 @@ class DataSourceController @Inject()(
       UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
       for {
         agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-        skeleton <- agglomerateService
-          .generateSkeleton(organizationId, datasetDirectoryName, dataLayerName, mappingName, agglomerateId)
-          .toFox ?~> "agglomerateSkeleton.failed"
+        skeleton <- agglomerateService.generateSkeleton(organizationId,
+                                                        datasetDirectoryName,
+                                                        dataLayerName,
+                                                        mappingName,
+                                                        agglomerateId) ?~> "agglomerateSkeleton.failed"
       } yield Ok(skeleton.toByteArray).as(protobufMimeType)
     }
   }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
index 76ab19b7e84..4efc89f0c81 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
@@ -3,6 +3,7 @@ package com.scalableminds.webknossos.datastore.datareaders
 import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.cache.AlfuCache
 import com.scalableminds.util.geometry.Vec3Int
+import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.datavault.VaultPath
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
@@ -201,7 +202,7 @@ class DatasetArray(vaultPath: VaultPath,
       val copiedFuture = Fox.combined(chunkIndices.map { chunkIndex: Array[Int] =>
         for {
           sourceChunk: MultiArray <- getSourceChunkDataWithCache(chunkIndex)
-          offsetInChunk = computeOffsetInChunkIgnoringAxisOrder(chunkIndex, totalOffset).reverse
+          offsetInChunk = computeOffsetInChunkIgnoringAxisOrder(chunkIndex, totalOffset)
           _ <- tryo(MultiArrayUtils.copyRange(offsetInChunk, sourceChunk, targetMultiArray)).toFox ?~> formatCopyRangeErrorWithoutAxisOrder(
             offsetInChunk,
             sourceChunk,
@@ -244,7 +245,6 @@ class DatasetArray(vaultPath: VaultPath,
     if (header.isSharded) {
       for {
         (shardPath, chunkRange) <- getShardedChunkPathAndRange(chunkIndex) ?~> "chunk.getShardedPathAndRange.failed"
-        _ = logger.info(s"chunk cache miss for $shardPath chunk ${chunkIndex.mkString(",")} ")
         chunkShape = chunkShapeAtIndex(chunkIndex)
         multiArray <- chunkReader.read(shardPath, chunkShape, Some(chunkRange), useSkipTypingShortcut)
       } yield multiArray
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index db34ed3c269..10eedf6e8b5 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -10,10 +10,10 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.AgglomerateGraph.{AgglomerateEdge, AgglomerateGraph}
 import com.scalableminds.webknossos.datastore.DataStoreConfig
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, SkeletonTracing, Tree, TreeTypeProto}
-import com.scalableminds.webknossos.datastore.datareaders.{AxisOrder, DatasetArray}
+import com.scalableminds.webknossos.datastore.datareaders.DatasetArray
 import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3Array
 import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto
-import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, SkeletonTracingDefaults}
+import com.scalableminds.webknossos.datastore.helpers.{NativeBucketScanner, NodeDefaults, SkeletonTracingDefaults}
 import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, ElementClass}
 import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
 import com.scalableminds.webknossos.datastore.storage._
@@ -21,7 +21,7 @@ import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.{Box, Failure, Full}
 import net.liftweb.common.Box.tryo
 import org.apache.commons.io.FilenameUtils
-import ucar.ma2.{Array => MultiArray}
+import ucar.ma2.{DataType, Index2D, Array => MultiArray}
 
 import java.net.URI
 import java.nio._
@@ -37,17 +37,16 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
     with LazyLogging {
   private val dataBaseDir = Paths.get(config.Datastore.baseDirectory)
   private val agglomerateDir = "agglomerates"
-  private val agglomerateFileExtension = ""
 
   private lazy val openArraysCache = AlfuCache[String, DatasetArray]()
 
-  // TODO unify with existing chunkContentsCache from binaryDataService
+  // TODO unify with existing chunkContentsCache from binaryDataService?
   private lazy val sharedChunkContentsCache: AlfuCache[String, MultiArray] = {
     // Used by DatasetArray-based datasets. Measure item weight in kilobytes because the weigher can only return int, not long
 
     val maxSizeKiloBytes = Math.floor(config.Datastore.Cache.ImageArrayChunks.maxSizeBytes.toDouble / 1000.0).toInt
 
-    def cacheWeight(key: String, arrayBox: Box[MultiArray]): Int =
+    def cacheWeight(_key: String, arrayBox: Box[MultiArray]): Int =
       arrayBox match {
         case Full(array) =>
           (array.getSizeBytes / 1000L).toInt
@@ -57,6 +56,8 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
     AlfuCache(maxSizeKiloBytes, weighFn = Some(cacheWeight))
   }
 
+  protected lazy val bucketScanner = new NativeBucketScanner()
+
   def readFromSegmentToAgglomerate(implicit ec: ExecutionContext, tc: TokenContext): Fox[ucar.ma2.Array] =
     for {
       zarrArray <- openZarrArrayCached("segment_to_agglomerate")
@@ -64,10 +65,10 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       _ = logger.info(s"read ${read.getSize} elements from agglomerate file segmentToAgglomerate")
     } yield read
 
-  private def mapSingleSegment(segmentId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Long] =
+  private def mapSingleSegment(zarrArray: DatasetArray, segmentId: Long)(implicit ec: ExecutionContext,
+                                                                         tc: TokenContext): Fox[Long] =
     for {
-      zarrArray <- openZarrArrayCached("segment_to_agglomerate")
-      asMultiArray <- zarrArray.readAsMultiArray(offset = Array(segmentId), shape = Array(1))
+      asMultiArray <- zarrArray.readAsMultiArray(shape = Array(1), offset = Array(segmentId))
     } yield asMultiArray.getLong(0)
 
   private def openZarrArrayCached(zarrArrayName: String)(implicit ec: ExecutionContext, tc: TokenContext) =
@@ -88,7 +89,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
                                    None,
                                    None,
                                    None,
-                                   sharedChunkContentsCache)(ec, TokenContext(None))
+                                   sharedChunkContentsCache)
     } yield zarrArray
   }
 
@@ -99,43 +100,126 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
     val zarrGroupPath = agglomerateFileKey.zarrGroupPath(dataBaseDir, agglomerateDir).toAbsolutePath
 
     def convertToAgglomerate(segmentIds: Array[Long],
+                             relevantAgglomerateMap: Map[Long, Long],
                              bytesPerElement: Int,
-                             putToBufferFunction: (ByteBuffer, Long) => ByteBuffer): Fox[Array[Byte]] =
-      for {
-        agglomerateIds <- Fox.serialCombined(segmentIds)(mapSingleSegment)
-        mappedBytes = agglomerateIds
-          .foldLeft(ByteBuffer.allocate(bytesPerElement * segmentIds.length).order(ByteOrder.LITTLE_ENDIAN))(
-            putToBufferFunction)
-          .array
-      } yield mappedBytes
+                             putToBufferFunction: (ByteBuffer, Long) => ByteBuffer): Array[Byte] = {
+      val agglomerateIds = segmentIds.map(relevantAgglomerateMap)
+      agglomerateIds
+        .foldLeft(ByteBuffer.allocate(bytesPerElement * segmentIds.length).order(ByteOrder.LITTLE_ENDIAN))(
+          putToBufferFunction)
+        .array
+    }
 
     val bytesPerElement = ElementClass.bytesPerElement(request.dataLayer.elementClass)
-    /* Every value of the segmentation data needs to be converted to Long to then look up the
-       agglomerate id in the segment-to-agglomerate array.
-       The value is first converted to the primitive signed number types, and then converted
-       to Long via uByteToLong, uShortToLong etc, which perform bitwise and to take care of
-       the unsigned semantics. Using functions avoids allocating intermediate SegmentInteger objects.
-       Allocating a fixed-length LongBuffer first is a further performance optimization.
-     */
-    convertData(data, request.dataLayer.elementClass) match {
-      case data: Array[Byte] =>
-        val longBuffer = LongBuffer.allocate(data.length)
-        data.foreach(e => longBuffer.put(uByteToLong(e)))
-        convertToAgglomerate(longBuffer.array, bytesPerElement, putByte)
-      case data: Array[Short] =>
-        val longBuffer = LongBuffer.allocate(data.length)
-        data.foreach(e => longBuffer.put(uShortToLong(e)))
-        convertToAgglomerate(longBuffer.array, bytesPerElement, putShort)
-      case data: Array[Int] =>
-        val longBuffer = LongBuffer.allocate(data.length)
-        data.foreach(e => longBuffer.put(uIntToLong(e)))
-        convertToAgglomerate(longBuffer.array, bytesPerElement, putInt)
-      case data: Array[Long] => convertToAgglomerate(data, bytesPerElement, putLong)
-      case _                 => Fox.successful(data)
-    }
+    val distinctSegmentIds =
+      bucketScanner.collectSegmentIds(data, bytesPerElement, isSigned = false, skipZeroes = false)
 
+    for {
+      zarrArray <- openZarrArrayCached("segment_to_agglomerate")
+      beforeBuildMap = Instant.now
+      relevantAgglomerateMap: Map[Long, Long] <- Fox
+        .serialCombined(distinctSegmentIds) { segmentId =>
+          mapSingleSegment(zarrArray, segmentId).map((segmentId, _))
+        }
+        .map(_.toMap)
+      _ = Instant.logSince(beforeBuildMap, "build map")
+      mappedBytes: Array[Byte] = convertData(data, request.dataLayer.elementClass) match {
+        case data: Array[Byte] =>
+          val longBuffer = LongBuffer.allocate(data.length)
+          data.foreach(e => longBuffer.put(uByteToLong(e)))
+          convertToAgglomerate(longBuffer.array, relevantAgglomerateMap, bytesPerElement, putByte)
+        case data: Array[Short] =>
+          val longBuffer = LongBuffer.allocate(data.length)
+          data.foreach(e => longBuffer.put(uShortToLong(e)))
+          convertToAgglomerate(longBuffer.array, relevantAgglomerateMap, bytesPerElement, putShort)
+        case data: Array[Int] =>
+          val longBuffer = LongBuffer.allocate(data.length)
+          data.foreach(e => longBuffer.put(uIntToLong(e)))
+          convertToAgglomerate(longBuffer.array, relevantAgglomerateMap, bytesPerElement, putInt)
+        case data: Array[Long] => convertToAgglomerate(data, relevantAgglomerateMap, bytesPerElement, putLong)
+        case _                 => data
+      }
+    } yield mappedBytes
   }
 
+  def generateSkeleton(organizationId: String,
+                       datasetDirectoryName: String,
+                       dataLayerName: String,
+                       mappingName: String,
+                       agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[SkeletonTracing] =
+    for {
+      before <- Instant.nowFox
+      agglomerate_to_segments_offsets <- openZarrArrayCached("agglomerate_to_segments_offsets")
+      agglomerate_to_edges_offsets <- openZarrArrayCached("agglomerate_to_edges_offsets")
+
+      positionsRange: MultiArray <- agglomerate_to_segments_offsets.readAsMultiArray(shape = Array(2),
+                                                                                     offset = Array(agglomerateId))
+      edgesRange: MultiArray <- agglomerate_to_edges_offsets.readAsMultiArray(shape = Array(2),
+                                                                              offset = Array(agglomerateId))
+      nodeCount = positionsRange.getLong(1) - positionsRange.getLong(0)
+      edgeCount = edgesRange.getLong(1) - edgesRange.getLong(0)
+      edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
+      _ <- Fox.fromBool(nodeCount <= edgeLimit) ?~> s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)"
+      _ <- Fox.fromBool(edgeCount <= edgeLimit) ?~> s"Agglomerate has too many edges ($edgeCount > $edgeLimit)"
+      positions: MultiArray <- if (nodeCount == 0L) {
+        Fox.successful(MultiArray.factory(DataType.LONG, Array(0, 0)))
+      } else {
+        for {
+          agglomerate_to_positions <- openZarrArrayCached("agglomerate_to_positions")
+          positions <- agglomerate_to_positions.readAsMultiArray(offset = Array(positionsRange.getLong(0), 0),
+                                                                 shape = Array(nodeCount.toInt, 3))
+        } yield positions
+      }
+      edges: MultiArray <- if (edgeCount == 0L) {
+        Fox.successful(MultiArray.factory(DataType.LONG, Array(0, 0)))
+      } else {
+        for {
+          agglomerate_to_edges <- openZarrArrayCached("agglomerate_to_edges")
+          edges <- agglomerate_to_edges.readAsMultiArray(offset = Array(edgesRange.getLong(0), 0),
+                                                         shape = Array(edgeCount.toInt, 2))
+        } yield edges
+      }
+
+      nodeIdStartAtOneOffset = 1
+
+      nodes = (0 until nodeCount.toInt).map { nodeIdx =>
+        NodeDefaults.createInstance.copy(
+          id = nodeIdx + nodeIdStartAtOneOffset,
+          position = Vec3IntProto(positions.getInt(new Index2D(Array(nodeIdx, 0))),
+                                  positions.getInt(new Index2D(Array(nodeIdx, 1))),
+                                  positions.getInt(new Index2D(Array(nodeIdx, 2))))
+        )
+      }
+
+      skeletonEdges = (0 until edges.getShape()(1)).map { edgeIdx =>
+        Edge(source = edges.getInt(new Index2D(Array(edgeIdx, 0))) + nodeIdStartAtOneOffset,
+             target = edges.getInt(new Index2D(Array(edgeIdx, 1))) + nodeIdStartAtOneOffset)
+      }
+
+      trees = Seq(
+        Tree(
+          treeId = math.abs(agglomerateId.toInt), // used only to deterministically select tree color
+          createdTimestamp = System.currentTimeMillis(),
+          // unsafeWrapArray is fine, because the underlying arrays are never mutated
+          nodes = nodes,
+          edges = skeletonEdges,
+          name = s"agglomerate $agglomerateId ($mappingName)",
+          `type` = Some(TreeTypeProto.AGGLOMERATE)
+        ))
+
+      skeleton = SkeletonTracingDefaults.createInstance.copy(
+        datasetName = datasetDirectoryName,
+        trees = trees
+      )
+
+      _ = if (Instant.since(before) > (100 milliseconds)) {
+        Instant.logSince(
+          before,
+          s"Generating skeleton from agglomerate file with ${skeletonEdges.length} edges, ${nodes.length} nodes",
+          logger)
+      }
+
+    } yield skeleton
 }
 
 class Hdf5AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
@@ -229,7 +313,7 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
     // We don't need to differentiate between the data types because the underlying library does the conversion for us
     reader.uint64().readArrayBlockWithOffset(hdf5Dataset, blockSize.toInt, segmentId)
 
-  // This uses the datasetDirectoryName, which allows us to call it on the same hdf file in parallel.
+  // This uses the datasetName, which allows us to call it on the same hdf file in parallel.
   private def readHDF(reader: IHDF5Reader, segmentId: Long, blockSize: Long) =
     reader.uint64().readArrayBlockWithOffset(datasetName, blockSize.toInt, segmentId)
 
@@ -268,89 +352,97 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
                        datasetDirectoryName: String,
                        dataLayerName: String,
                        mappingName: String,
-                       agglomerateId: Long): Box[SkeletonTracing] =
-    try {
-      val before = Instant.now
-      val hdfFile =
-        dataBaseDir
-          .resolve(organizationId)
-          .resolve(datasetDirectoryName)
-          .resolve(dataLayerName)
-          .resolve(agglomerateDir)
-          .resolve(s"$mappingName.$agglomerateFileExtension")
-          .toFile
-
-      val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
-      val positionsRange: Array[Long] =
-        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
-      val edgesRange: Array[Long] =
-        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
-
-      val nodeCount = positionsRange(1) - positionsRange(0)
-      val edgeCount = edgesRange(1) - edgesRange(0)
-      val edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
-      if (nodeCount > edgeLimit) {
-        throw new Exception(s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)")
-      }
-      if (edgeCount > edgeLimit) {
-        throw new Exception(s"Agglomerate has too many edges ($edgeCount > $edgeLimit)")
-      }
-      val positions: Array[Array[Long]] =
-        if (nodeCount == 0L) {
-          Array.empty[Array[Long]]
-        } else {
-          reader
-            .uint64()
-            .readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
+                       agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[SkeletonTracing] =
+    if (true) {
+      zarrAgglomerateService.generateSkeleton(organizationId,
+                                              datasetDirectoryName,
+                                              dataLayerName,
+                                              mappingName,
+                                              agglomerateId)
+    } else {
+      (try {
+        val before = Instant.now
+        val hdfFile =
+          dataBaseDir
+            .resolve(organizationId)
+            .resolve(datasetDirectoryName)
+            .resolve(dataLayerName)
+            .resolve(agglomerateDir)
+            .resolve(s"$mappingName.$agglomerateFileExtension")
+            .toFile
+
+        val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+        val positionsRange: Array[Long] =
+          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
+        val edgesRange: Array[Long] =
+          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
+
+        val nodeCount = positionsRange(1) - positionsRange(0)
+        val edgeCount = edgesRange(1) - edgesRange(0)
+        val edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
+        if (nodeCount > edgeLimit) {
+          throw new Exception(s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)")
         }
-      val edges: Array[Array[Long]] = {
-        if (edgeCount == 0L) {
-          Array.empty[Array[Long]]
-        } else {
-          reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
+        if (edgeCount > edgeLimit) {
+          throw new Exception(s"Agglomerate has too many edges ($edgeCount > $edgeLimit)")
+        }
+        val positions: Array[Array[Long]] =
+          if (nodeCount == 0L) {
+            Array.empty[Array[Long]]
+          } else {
+            reader
+              .uint64()
+              .readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
+          }
+        val edges: Array[Array[Long]] = {
+          if (edgeCount == 0L) {
+            Array.empty[Array[Long]]
+          } else {
+            reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
+          }
         }
-      }
-
-      val nodeIdStartAtOneOffset = 1
-
-      val nodes = positions.zipWithIndex.map {
-        case (pos, idx) =>
-          NodeDefaults.createInstance.copy(
-            id = idx + nodeIdStartAtOneOffset,
-            position = Vec3IntProto(pos(0).toInt, pos(1).toInt, pos(2).toInt)
-          )
-      }
 
-      val skeletonEdges = edges.map { e =>
-        Edge(source = e(0).toInt + nodeIdStartAtOneOffset, target = e(1).toInt + nodeIdStartAtOneOffset)
-      }
+        val nodeIdStartAtOneOffset = 1
 
-      val trees = Seq(
-        Tree(
-          treeId = math.abs(agglomerateId.toInt), // used only to deterministically select tree color
-          createdTimestamp = System.currentTimeMillis(),
-          // unsafeWrapArray is fine, because the underlying arrays are never mutated
-          nodes = ArraySeq.unsafeWrapArray(nodes),
-          edges = ArraySeq.unsafeWrapArray(skeletonEdges),
-          name = s"agglomerate $agglomerateId ($mappingName)",
-          `type` = Some(TreeTypeProto.AGGLOMERATE)
-        ))
+        val nodes = positions.zipWithIndex.map {
+          case (pos, idx) =>
+            NodeDefaults.createInstance.copy(
+              id = idx + nodeIdStartAtOneOffset,
+              position = Vec3IntProto(pos(0).toInt, pos(1).toInt, pos(2).toInt)
+            )
+        }
 
-      val skeleton = SkeletonTracingDefaults.createInstance.copy(
-        datasetName = datasetDirectoryName,
-        trees = trees
-      )
+        val skeletonEdges = edges.map { e =>
+          Edge(source = e(0).toInt + nodeIdStartAtOneOffset, target = e(1).toInt + nodeIdStartAtOneOffset)
+        }
 
-      if (Instant.since(before) > (100 milliseconds)) {
-        Instant.logSince(
-          before,
-          s"Generating skeleton from agglomerate file with ${skeletonEdges.length} edges, ${nodes.length} nodes",
-          logger)
-      }
+        val trees = Seq(
+          Tree(
+            treeId = math.abs(agglomerateId.toInt), // used only to deterministically select tree color
+            createdTimestamp = System.currentTimeMillis(),
+            // unsafeWrapArray is fine, because the underlying arrays are never mutated
+            nodes = ArraySeq.unsafeWrapArray(nodes),
+            edges = ArraySeq.unsafeWrapArray(skeletonEdges),
+            name = s"agglomerate $agglomerateId ($mappingName)",
+            `type` = Some(TreeTypeProto.AGGLOMERATE)
+          ))
+
+        val skeleton = SkeletonTracingDefaults.createInstance.copy(
+          datasetName = datasetDirectoryName,
+          trees = trees
+        )
+
+        if (Instant.since(before) > (100 milliseconds)) {
+          Instant.logSince(
+            before,
+            s"Generating skeleton from agglomerate file with ${skeletonEdges.length} edges, ${nodes.length} nodes",
+            logger)
+        }
 
-      Full(skeleton)
-    } catch {
-      case e: Exception => Failure(e.getMessage)
+        Full(skeleton)
+      } catch {
+        case e: Exception => Failure(e.getMessage)
+      }).toFox
     }
 
   def largestAgglomerateId(agglomerateFileKey: AgglomerateFileKey): Box[Long] = {

From 56ce08ba40f59cacb42290dc8320fd0cb5e88538 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 27 May 2025 13:53:58 +0200
Subject: [PATCH 10/22] fix reading agglomerate skeleton

---
 .../datastore/datareaders/DatasetArray.scala  |  2 +-
 .../services/AgglomerateService.scala         | 19 ++++++++++++-------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
index 4efc89f0c81..c09f0500d75 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
@@ -198,7 +198,7 @@ class DatasetArray(vaultPath: VaultPath,
       } yield sourceChunk
     } else {
       val targetBuffer = MultiArrayUtils.createDataBuffer(header.resolvedDataType, shape)
-      val targetMultiArray = MultiArrayUtils.createArrayWithGivenStorage(targetBuffer, shape.reverse)
+      val targetMultiArray = MultiArrayUtils.createArrayWithGivenStorage(targetBuffer, shape)
       val copiedFuture = Fox.combined(chunkIndices.map { chunkIndex: Array[Int] =>
         for {
           sourceChunk: MultiArray <- getSourceChunkDataWithCache(chunkIndex)
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 10eedf6e8b5..7fed7a1a999 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -21,7 +21,7 @@ import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.{Box, Failure, Full}
 import net.liftweb.common.Box.tryo
 import org.apache.commons.io.FilenameUtils
-import ucar.ma2.{DataType, Index2D, Array => MultiArray}
+import ucar.ma2.{DataType, Array => MultiArray}
 
 import java.net.URI
 import java.nio._
@@ -182,18 +182,23 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
 
       nodeIdStartAtOneOffset = 1
 
+      // TODO use multiarray index iterators?
       nodes = (0 until nodeCount.toInt).map { nodeIdx =>
         NodeDefaults.createInstance.copy(
           id = nodeIdx + nodeIdStartAtOneOffset,
-          position = Vec3IntProto(positions.getInt(new Index2D(Array(nodeIdx, 0))),
-                                  positions.getInt(new Index2D(Array(nodeIdx, 1))),
-                                  positions.getInt(new Index2D(Array(nodeIdx, 2))))
+          position = Vec3IntProto(
+            positions.getInt(positions.getIndex.set(Array(nodeIdx, 0))),
+            positions.getInt(positions.getIndex.set(Array(nodeIdx, 1))),
+            positions.getInt(positions.getIndex.set(Array(nodeIdx, 2)))
+          )
         )
       }
 
-      skeletonEdges = (0 until edges.getShape()(1)).map { edgeIdx =>
-        Edge(source = edges.getInt(new Index2D(Array(edgeIdx, 0))) + nodeIdStartAtOneOffset,
-             target = edges.getInt(new Index2D(Array(edgeIdx, 1))) + nodeIdStartAtOneOffset)
+      skeletonEdges = (0 until edges.getShape()(0)).map { edgeIdx =>
+        Edge(
+          source = edges.getInt(edges.getIndex.set(Array(edgeIdx, 0))) + nodeIdStartAtOneOffset,
+          target = edges.getInt(edges.getIndex.set(Array(edgeIdx, 1))) + nodeIdStartAtOneOffset
+        )
       }
 
       trees = Seq(

From 2855d2b31468c88c7fd09550f0fd2f1a6635ab89 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 27 May 2025 14:20:04 +0200
Subject: [PATCH 11/22] Change DatasetArray shape from Int to Long. Implement
 reading largestAgglomerateId

---
 .../datastore/dataformats/wkw/WKWHeader.scala |  2 +-
 .../datastore/datareaders/AxisOrder.scala     |  3 +++
 .../datastore/datareaders/ChunkUtils.scala    |  8 ++++----
 .../datastore/datareaders/DatasetArray.scala  | 19 +++++++++++--------
 .../datastore/datareaders/DatasetHeader.scala | 10 +++++++---
 .../datastore/datareaders/n5/N5Header.scala   |  4 ++--
 .../precomputed/PrecomputedHeader.scala       |  6 +++---
 .../datastore/datareaders/wkw/WKWArray.scala  |  2 +-
 .../datareaders/zarr/ZarrHeader.scala         |  6 +++---
 .../datareaders/zarr3/Zarr3ArrayHeader.scala  |  8 ++++----
 .../explore/NgffExplorationUtils.scala        |  6 +++---
 .../datastore/explore/NgffV0_4Explorer.scala  |  4 ++--
 .../datastore/explore/NgffV0_5Explorer.scala  |  2 +-
 .../explore/PrecomputedExplorer.scala         |  2 +-
 .../services/AgglomerateService.scala         | 12 +++++++++++-
 ...VolumeTracingZarrStreamingController.scala |  6 +++---
 16 files changed, 60 insertions(+), 40 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWHeader.scala
index d694ef96163..adde6bb2817 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWHeader.scala
@@ -78,7 +78,7 @@ case class WKWHeader(
     }
   }
 
-  override def datasetShape: Option[Array[Int]] = None
+  override def datasetShape: Option[Array[Long]] = None
 
   override def chunkShape: Array[Int] =
     Array(numChannels, numVoxelsPerChunkDimension, numVoxelsPerChunkDimension, numVoxelsPerChunkDimension)
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala
index 809d4d5e5cf..b8deb810cd6 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala
@@ -96,6 +96,9 @@ case class FullAxisOrder(axes: Seq[Axis]) {
   def permuteIndicesArrayToWk(indices: Array[Int]): Array[Int] =
     arrayToWkPermutation.map(indices(_))
 
+  def permuteIndicesArrayToWkLong(indices: Array[Long]): Array[Long] =
+    arrayToWkPermutation.map(indices(_))
+
   def toWkLibsJson: JsValue =
     Json.toJson(axes.zipWithIndex.collect {
       case (axis, index) if axis.name == "x" || axis.name == "y" || axis.name == "z" =>
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala
index f2917923e4a..8959b15e2bc 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala
@@ -1,16 +1,16 @@
 package com.scalableminds.webknossos.datastore.datareaders
 
 object ChunkUtils {
-  def computeChunkIndices(arrayShapeOpt: Option[Array[Int]],
+  def computeChunkIndices(arrayShapeOpt: Option[Array[Long]],
                           arrayChunkShape: Array[Int],
                           selectedShape: Array[Int],
-                          selectedOffset: Array[Long]): List[Array[Int]] = {
+                          selectedOffset: Array[Long]): Seq[Array[Int]] = {
     val nDims = arrayChunkShape.length
     val start = new Array[Int](nDims)
     val end = new Array[Int](nDims)
     var numChunks = 1
     for (dim <- 0 until nDims) {
-      val largestPossibleIndex = arrayShapeOpt.map(arrayShape => (arrayShape(dim) - 1) / arrayChunkShape(dim))
+      val largestPossibleIndex = arrayShapeOpt.map(arrayShape => ((arrayShape(dim) - 1) / arrayChunkShape(dim)).toInt)
       val smallestPossibleIndex = 0
       val startIndexRaw = (selectedOffset(dim) / arrayChunkShape(dim)).toInt
       val startIndexClamped =
@@ -38,6 +38,6 @@ object ChunkUtils {
         dimIndex = -1
       }
     }
-    chunkIndices.toList
+    chunkIndices.toSeq
   }
 }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
index c09f0500d75..c4b5309caab 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
@@ -51,7 +51,7 @@ class DatasetArray(vaultPath: VaultPath,
     header.rank + 1
   }
 
-  lazy val datasetShape: Option[Array[Int]] = if (axisOrder.hasZAxis) {
+  lazy val datasetShape: Option[Array[Long]] = if (axisOrder.hasZAxis) {
     header.datasetShape
   } else {
     header.datasetShape.map(shape => shape :+ 1)
@@ -156,10 +156,12 @@ class DatasetArray(vaultPath: VaultPath,
   private def readAsFortranOrder(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext,
                                                                         tc: TokenContext): Fox[MultiArray] = {
     val totalOffset: Array[Int] = offset.zip(header.voxelOffset).map { case (o, v) => o - v }.padTo(offset.length, 0)
-    val chunkIndices = ChunkUtils.computeChunkIndices(datasetShape.map(fullAxisOrder.permuteIndicesArrayToWk),
-                                                      fullAxisOrder.permuteIndicesArrayToWk(chunkShape),
-                                                      shape,
-                                                      totalOffset.map(_.toLong))
+    val chunkIndices = ChunkUtils.computeChunkIndices(
+      datasetShape.map(fullAxisOrder.permuteIndicesArrayToWkLong),
+      fullAxisOrder.permuteIndicesArrayToWk(chunkShape),
+      shape,
+      totalOffset.map(_.toLong)
+    )
     if (partialCopyingIsNotNeededForWkOrder(shape, totalOffset, chunkIndices)) {
       for {
         chunkIndex <- chunkIndices.headOption.toFox
@@ -263,7 +265,7 @@ class DatasetArray(vaultPath: VaultPath,
 
   private def partialCopyingIsNotNeededForMultiArray(bufferShape: Array[Int],
                                                      globalOffset: Array[Long],
-                                                     chunkIndices: List[Array[Int]]): Boolean =
+                                                     chunkIndices: Seq[Array[Int]]): Boolean =
     chunkIndices match {
       case chunkIndex :: Nil =>
         val offsetInChunk = computeOffsetInChunkIgnoringAxisOrder(chunkIndex, globalOffset)
@@ -274,7 +276,7 @@ class DatasetArray(vaultPath: VaultPath,
 
   private def partialCopyingIsNotNeededForWkOrder(bufferShape: Array[Int],
                                                   globalOffset: Array[Int],
-                                                  chunkIndices: List[Array[Int]]): Boolean =
+                                                  chunkIndices: Seq[Array[Int]]): Boolean =
     chunkIndices match {
       case chunkIndex :: Nil =>
         val offsetInChunk = computeOffsetInChunk(chunkIndex, globalOffset)
@@ -301,8 +303,9 @@ class DatasetArray(vaultPath: VaultPath,
       (globalOffset(dim) - (chunkIndex(dim).toLong * chunkShape(dim).toLong)).toInt
     }.toArray
 
+  // TODO works only for wk dataet arrays, not agglomerate files
   override def toString: String =
-    s"${getClass.getCanonicalName} fullAxisOrder=$fullAxisOrder shape=${header.datasetShape.map(s => printAsInner(s))} chunkShape=${printAsInner(
+    s"${getClass.getCanonicalName} fullAxisOrder=$fullAxisOrder shape=${header.datasetShape.map(s => printAsInner(s.map(_.toInt)))} chunkShape=${printAsInner(
       header.chunkShape)} dtype=${header.resolvedDataType} fillValue=${header.fillValueNumber}, ${header.compressorImpl}, byteOrder=${header.byteOrder}, vault=${vaultPath.summary}}"
 
 }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
index 6907d49ecae..3935b42ea6b 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
@@ -11,7 +11,7 @@ import java.nio.ByteOrder
 trait DatasetHeader {
 
   // Note that in DatasetArray, datasetShape and chunkShape are adapted for 2d datasets
-  def datasetShape: Option[Array[Int]] // shape of the entire array
+  def datasetShape: Option[Array[Long]] // shape of the entire array
   def chunkShape: Array[Int] // shape of each chunk,
 
   def dimension_separator: DimensionSeparator
@@ -44,9 +44,13 @@ trait DatasetHeader {
         None
       else {
         if (axisOrder.hasZAxis) {
-          Some(BoundingBox(Vec3Int.zeros, shape(axisOrder.x), shape(axisOrder.y), shape(axisOrder.zWithFallback)))
+          Some(
+            BoundingBox(Vec3Int.zeros,
+                        shape(axisOrder.x).toInt,
+                        shape(axisOrder.y).toInt,
+                        shape(axisOrder.zWithFallback).toInt))
         } else {
-          Some(BoundingBox(Vec3Int.zeros, shape(axisOrder.x), shape(axisOrder.y), 1))
+          Some(BoundingBox(Vec3Int.zeros, shape(axisOrder.x).toInt, shape(axisOrder.y).toInt, 1))
         }
       }
     }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5Header.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5Header.scala
index 7cc5542b940..8e178636971 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5Header.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5Header.scala
@@ -16,7 +16,7 @@ object N5BlockHeader {
 }
 
 case class N5Header(
-    dimensions: Array[Int], // shape of the entire array
+    dimensions: Array[Long], // shape of the entire array
     blockSize: Array[Int], // shape of each chunk
     compression: Option[Map[String, CompressionSetting]] = None, // specifies compressor to use, with parameters
     dataType: String,
@@ -25,7 +25,7 @@ case class N5Header(
   val fill_value: Either[String, Number] = Right(0)
   val order: ArrayOrder = ArrayOrder.F
 
-  override lazy val datasetShape: Option[Array[Int]] = Some(dimensions)
+  override lazy val datasetShape: Option[Array[Long]] = Some(dimensions)
 
   lazy val chunkShape: Array[Int] = blockSize
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala
index 430547d5ea0..ac05bd26556 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala
@@ -28,7 +28,7 @@ case class PrecomputedHeader(`type`: String,
 }
 
 case class PrecomputedScale(key: String,
-                            size: Array[Int],
+                            size: Array[Long],
                             resolution: Array[Double],
                             chunk_sizes: Array[Array[Int]],
                             encoding: String,
@@ -45,7 +45,7 @@ case class PrecomputedScale(key: String,
 
 case class PrecomputedScaleHeader(precomputedScale: PrecomputedScale, precomputedHeader: PrecomputedHeader)
     extends DatasetHeader {
-  override def datasetShape: Option[Array[Int]] = Some(precomputedScale.size)
+  override def datasetShape: Option[Array[Long]] = Some(precomputedScale.size)
 
   override def chunkShape: Array[Int] = precomputedScale.chunk_sizes.head
 
@@ -72,7 +72,7 @@ case class PrecomputedScaleHeader(precomputedScale: PrecomputedScale, precompute
       val (chunkIndexAtDim, dim) = chunkIndexWithDim
       val beginOffset = voxelOffset(dim) + chunkIndexAtDim * precomputedScale.primaryChunkShape(dim)
       val endOffset = voxelOffset(dim) + ((chunkIndexAtDim + 1) * precomputedScale.primaryChunkShape(dim))
-        .min(precomputedScale.size(dim))
+        .min(precomputedScale.size(dim).toInt)
       (beginOffset, endOffset)
     })
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
index f5e7232f9f1..31242525413 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
@@ -129,7 +129,7 @@ class WKWArray(vaultPath: VaultPath,
 
   private def chunkIndexToShardIndex(chunkIndex: Array[Int]) =
     ChunkUtils.computeChunkIndices(
-      header.datasetShape.map(fullAxisOrder.permuteIndicesArrayToWk),
+      header.datasetShape.map(fullAxisOrder.permuteIndicesArrayToWkLong),
       fullAxisOrder.permuteIndicesArrayToWk(header.shardShape),
       header.chunkShape,
       chunkIndex.zip(header.chunkShape).map { case (i, s) => i * s }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/ZarrHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/ZarrHeader.scala
index 66fe7090deb..6aaf01e431e 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/ZarrHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/ZarrHeader.scala
@@ -21,7 +21,7 @@ import play.api.libs.json._
 
 case class ZarrHeader(
     zarr_format: Int, // format version number
-    shape: Array[Int], // shape of the entire array
+    shape: Array[Long], // shape of the entire array
     chunks: Array[Int], // shape of each chunk
     compressor: Option[Map[String, CompressionSetting]] = None, // specifies compressor to use, with parameters
     filters: Option[List[Map[String, String]]] = None, // specifies filters to use, with parameters
@@ -31,7 +31,7 @@ case class ZarrHeader(
     override val order: ArrayOrder
 ) extends DatasetHeader {
 
-  override lazy val datasetShape: Option[Array[Int]] = Some(shape)
+  override lazy val datasetShape: Option[Array[Long]] = Some(shape)
   override lazy val chunkShape: Array[Int] = chunks
 
   override lazy val byteOrder: ByteOrder =
@@ -77,7 +77,7 @@ object ZarrHeader extends JsonImplicits {
     val chunks = Array(channels) ++ additionalAxesChunksEntries ++ Array(cubeLength, cubeLength, cubeLength)
 
     ZarrHeader(zarr_format = 2,
-               shape = shape,
+               shape = shape.map(_.toLong),
                chunks = chunks,
                compressor = compressor,
                dtype = dtype,
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala
index 262b462abf0..adee8ddcd2c 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala
@@ -25,7 +25,7 @@ import java.nio.ByteOrder
 case class Zarr3ArrayHeader(
     zarr_format: Int, // must be 3
     node_type: String, // must be "array"
-    shape: Array[Int],
+    shape: Array[Long],
     data_type: Either[String, ExtensionDataType],
     chunk_grid: Either[ChunkGridSpecification, ExtensionChunkGridSpecification],
     chunk_key_encoding: ChunkKeyEncoding,
@@ -36,7 +36,7 @@ case class Zarr3ArrayHeader(
     dimension_names: Option[Array[String]]
 ) extends DatasetHeader {
 
-  override def datasetShape: Option[Array[Int]] = Some(shape)
+  override def datasetShape: Option[Array[Long]] = Some(shape)
 
   override def chunkShape: Array[Int] = getChunkSize
 
@@ -168,7 +168,7 @@ object Zarr3ArrayHeader extends JsonImplicits {
       for {
         zarr_format <- (json \ "zarr_format").validate[Int]
         node_type <- (json \ "node_type").validate[String]
-        shape <- (json \ "shape").validate[Array[Int]]
+        shape <- (json \ "shape").validate[Array[Long]]
         data_type <- (json \ "data_type").validate[String]
         chunk_grid <- (json \ "chunk_grid").validate[ChunkGridSpecification]
         chunk_key_encoding <- (json \ "chunk_key_encoding").validate[ChunkKeyEncoding]
@@ -271,7 +271,7 @@ object Zarr3ArrayHeader extends JsonImplicits {
       zarr_format = 3,
       node_type = "array",
       // channel, additional axes, XYZ
-      shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ xyzBBounds,
+      shape = (Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ xyzBBounds).map(_.toLong),
       data_type = Left(dataLayer.elementClass.toString),
       chunk_grid = Left(
         ChunkGridSpecification(
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
index c62c1ddd5b2..32a67a0d798 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
@@ -182,7 +182,7 @@ trait NgffExplorationUtils extends FoxImplicits {
     Vec3Double(xFactors.product, yFactors.product, zFactors.product)
   }
 
-  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Int]]
+  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Long]]
 
   protected def createAdditionalAxis(name: String, index: Int, bounds: Array[Int]): Box[AdditionalAxis] =
     for {
@@ -203,7 +203,7 @@ trait NgffExplorationUtils extends FoxImplicits {
           .filter(axis => !defaultAxes.contains(axis.name))
           .zipWithIndex
           .map(axisAndIndex =>
-            createAdditionalAxis(axisAndIndex._1.name, axisAndIndex._2, Array(0, shape(axisAndIndex._2))).toFox))
+            createAdditionalAxis(axisAndIndex._1.name, axisAndIndex._2, Array(0, shape(axisAndIndex._2).toInt)).toFox))
       duplicateNames = axes.map(_.name).diff(axes.map(_.name).distinct).distinct
       _ <- Fox.fromBool(duplicateNames.isEmpty) ?~> s"Additional axes names (${duplicateNames.mkString("", ", ", "")}) are not unique."
     } yield axes
@@ -220,7 +220,7 @@ trait NgffExplorationUtils extends FoxImplicits {
         case Some(channeAxislIndex) => shape(channeAxislIndex)
         case _                      => 1
       }
-    } yield channelCount
+    } yield channelCount.toInt
 
   protected def createLayer(remotePath: VaultPath,
                             credentialId: Option[String],
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_4Explorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_4Explorer.scala
index 7f6991e1ae3..9b40b90427f 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_4Explorer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_4Explorer.scala
@@ -96,7 +96,7 @@ class NgffV0_4Explorer(implicit val ec: ExecutionContext)
       parsedHeader <- zarrayPath.parseAsJson[ZarrHeader] ?~> s"failed to read zarr header at $zarrayPath"
       header = parsedHeader.shape.length match {
         case 2 =>
-          parsedHeader.copy(shape = parsedHeader.shape ++ Array(1), chunks = parsedHeader.chunks ++ Array(1))
+          parsedHeader.copy(shape = parsedHeader.shape ++ Array(1L), chunks = parsedHeader.chunks ++ Array(1))
         case _ => parsedHeader
       }
     } yield header
@@ -125,7 +125,7 @@ class NgffV0_4Explorer(implicit val ec: ExecutionContext)
         elementClass,
         boundingBox)
 
-  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Int]] =
+  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Long]] =
     for {
       zarrHeader <- getZarrHeader(dataset, path)
       shape = zarrHeader.shape
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_5Explorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_5Explorer.scala
index 3b67e6902bb..6ec2421e76a 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_5Explorer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_5Explorer.scala
@@ -122,7 +122,7 @@ class NgffV0_5Explorer(implicit val ec: ExecutionContext)
         elementClass,
         boundingBox)
 
-  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Int]] =
+  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Long]] =
     for {
       zarrHeader <- getZarrHeader(dataset, path)
       shape = zarrHeader.shape
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/PrecomputedExplorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/PrecomputedExplorer.scala
index 02f15f564a7..7077ccc5e3f 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/PrecomputedExplorer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/PrecomputedExplorer.scala
@@ -36,7 +36,7 @@ class PrecomputedExplorer(implicit val ec: ExecutionContext) extends RemoteLayer
       name <- Fox.successful(guessNameFromPath(remotePath))
       firstScale <- precomputedHeader.scales.headOption.toFox
       boundingBox <- BoundingBox
-        .fromTopLeftAndSize(firstScale.voxel_offset.getOrElse(Array(0, 0, 0)), firstScale.size)
+        .fromTopLeftAndSize(firstScale.voxel_offset.getOrElse(Array(0, 0, 0)), firstScale.size.map(_.toInt))
         .toFox
       elementClass: ElementClass.Value <- elementClassFromPrecomputedDataType(precomputedHeader.data_type).toFox ?~> s"Unknown data type ${precomputedHeader.data_type}"
       smallestResolution = firstScale.resolution
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 7fed7a1a999..43cb8e85930 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -38,6 +38,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
   private val dataBaseDir = Paths.get(config.Datastore.baseDirectory)
   private val agglomerateDir = "agglomerates"
 
+  // TODO clear on dataset reload
   private lazy val openArraysCache = AlfuCache[String, DatasetArray]()
 
   // TODO unify with existing chunkContentsCache from binaryDataService?
@@ -46,7 +47,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
 
     val maxSizeKiloBytes = Math.floor(config.Datastore.Cache.ImageArrayChunks.maxSizeBytes.toDouble / 1000.0).toInt
 
-    def cacheWeight(_key: String, arrayBox: Box[MultiArray]): Int =
+    def cacheWeight(key: String, arrayBox: Box[MultiArray]): Int =
       arrayBox match {
         case Full(array) =>
           (array.getSizeBytes / 1000L).toInt
@@ -225,6 +226,15 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       }
 
     } yield skeleton
+
+  def largestAgglomerateId(agglomerateFileKey: AgglomerateFileKey)(implicit ec: ExecutionContext,
+                                                                   tc: TokenContext): Fox[Long] =
+    for {
+      array <- openZarrArrayCached("agglomerate_to_segments_offsets")
+      shape <- array.datasetShape.toFox ?~> "Could not determine array shape"
+      shapeFirstElement <- tryo(shape(0)).toFox
+    } yield shapeFirstElement
+
 }
 
 class Hdf5AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala
index 8330a386eb3..9c24db0cb66 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala
@@ -159,7 +159,7 @@ class VolumeTracingZarrStreamingController @Inject()(
           chunks = Array(channels, cubeLength, cubeLength, cubeLength)
 
           zarrHeader = ZarrHeader(zarr_format = 2,
-                                  shape = shape,
+                                  shape = shape.map(_.toLong),
                                   chunks = chunks,
                                   compressor = compressor,
                                   dtype = dtype,
@@ -188,11 +188,11 @@ class VolumeTracingZarrStreamingController @Inject()(
             zarr_format = 3,
             node_type = "array",
             // channel, additional axes, XYZ
-            shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ Array(
+            shape = (Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ Array(
               (tracing.boundingBox.width + tracing.boundingBox.topLeft.x) / magParsed.x,
               (tracing.boundingBox.height + tracing.boundingBox.topLeft.y) / magParsed.y,
               (tracing.boundingBox.depth + tracing.boundingBox.topLeft.z) / magParsed.z
-            ),
+            )).map(_.toLong),
             data_type = Left(tracing.elementClass.toString),
             chunk_grid = Left(
               ChunkGridSpecification(

From 4ed54836428fbceffc4f01d813ad2807f940a7fb Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 27 May 2025 14:27:07 +0200
Subject: [PATCH 12/22] remove unused agglomeratesForAllSegments

---
 .../controllers/DataSourceController.scala    | 24 -------------------
 .../services/AgglomerateService.scala         |  9 -------
 ....scalableminds.webknossos.datastore.routes |  1 -
 3 files changed, 34 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
index c7bcdb229de..b49ebb2bd72 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
@@ -374,30 +374,6 @@ class DataSourceController @Inject()(
     }
   }
 
-  def agglomerateIdsForAllSegmentIds(
-      organizationId: String,
-      datasetDirectoryName: String,
-      dataLayerName: String,
-      mappingName: String
-  ): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request =>
-    accessTokenService.validateAccessFromTokenContext(
-      UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
-      for {
-        agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-        agglomerateIds: Array[Long] <- agglomerateService
-          .agglomerateIdsForAllSegmentIds(
-            AgglomerateFileKey(
-              organizationId,
-              datasetDirectoryName,
-              dataLayerName,
-              mappingName
-            )
-          )
-          .toFox
-      } yield Ok(Json.toJson(agglomerateIds))
-    }
-  }
-
   def update(organizationId: String, datasetDirectoryName: String): Action[DataSource] =
     Action.async(validateJson[DataSource]) { implicit request =>
       accessTokenService.validateAccessFromTokenContext(
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 43cb8e85930..88d497c7fb3 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -509,15 +509,6 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
 
   }
 
-  def agglomerateIdsForAllSegmentIds(agglomerateFileKey: AgglomerateFileKey): Box[Array[Long]] = {
-    val file = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
-    tryo {
-      val reader = HDF5FactoryProvider.get.openForReading(file)
-      val agglomerateIds: Array[Long] = reader.uint64().readArray("/segment_to_agglomerate")
-      agglomerateIds
-    }
-  }
-
   def positionForSegmentId(agglomerateFileKey: AgglomerateFileKey, segmentId: Long): Box[Vec3Int] = {
     val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
     val reader: IHDF5Reader = HDF5FactoryProvider.get.openForReading(hdfFile)
diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
index ea1aaa70266..bcaf26dc3f6 100644
--- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
+++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
@@ -76,7 +76,6 @@ GET           /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerN
 GET           /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/agglomerates/:mappingName/agglomerateGraph/:agglomerateId         @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateGraph(organizationId: String, datasetDirectoryName: String, dataLayerName: String, mappingName: String, agglomerateId: Long)
 GET           /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/agglomerates/:mappingName/largestAgglomerateId                    @com.scalableminds.webknossos.datastore.controllers.DataSourceController.largestAgglomerateId(organizationId: String, datasetDirectoryName: String, dataLayerName: String, mappingName: String)
 POST          /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/agglomerates/:mappingName/agglomeratesForSegments                 @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateIdsForSegmentIds(organizationId: String, datasetDirectoryName: String, dataLayerName: String, mappingName: String)
-GET           /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/agglomerates/:mappingName/agglomeratesForAllSegments              @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateIdsForAllSegmentIds(organizationId: String, datasetDirectoryName: String, dataLayerName: String, mappingName: String)
 GET           /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/agglomerates/:mappingName/positionForSegment                      @com.scalableminds.webknossos.datastore.controllers.DataSourceController.positionForSegmentViaAgglomerateFile(organizationId: String, datasetDirectoryName: String, dataLayerName: String, mappingName: String, segmentId: Long)
 
 # Mesh files

From 291aab5ed133fd15866855d6582501a6e2fda5c8 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 27 May 2025 16:14:12 +0200
Subject: [PATCH 13/22] add shortcut for shape.product==0; implement
 segmentIdsForAgglomerateId

---
 .../controllers/DSMeshController.scala        |   2 +-
 .../controllers/DataSourceController.scala    |  24 +-
 .../datastore/datareaders/DatasetArray.scala  |  49 ++--
 .../datareaders/MultiArrayUtils.scala         |   4 +
 .../services/AgglomerateService.scala         | 267 +++++++++++-------
 .../services/SegmentIndexFileService.scala    |  33 ++-
 .../services/mesh/MeshFileService.scala       |  15 +-
 .../services/mesh/MeshMappingHelper.scala     |  42 +--
 ...uroglancerPrecomputedMeshFileService.scala |   2 +-
 9 files changed, 252 insertions(+), 186 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala
index 9bc8d02096a..9d838421d4e 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala
@@ -66,7 +66,7 @@ class DSMeshController @Inject()(
                                                                           datasetDirectoryName,
                                                                           dataLayerName,
                                                                           request.body.meshFile.name)
-          segmentIds: List[Long] <- segmentIdsForAgglomerateIdIfNeeded(
+          segmentIds: Seq[Long] <- segmentIdsForAgglomerateIdIfNeeded(
             organizationId,
             datasetDirectoryName,
             dataLayerName,
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
index b49ebb2bd72..06508d7c469 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
@@ -297,11 +297,9 @@ class DataSourceController @Inject()(
       UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
       for {
         agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-        agglomerateGraph <- agglomerateService
-          .generateAgglomerateGraph(
-            AgglomerateFileKey(organizationId, datasetDirectoryName, dataLayerName, mappingName),
-            agglomerateId)
-          .toFox ?~> "agglomerateGraph.failed"
+        agglomerateGraph <- agglomerateService.generateAgglomerateGraph(
+          AgglomerateFileKey(organizationId, datasetDirectoryName, dataLayerName, mappingName),
+          agglomerateId) ?~> "agglomerateGraph.failed"
       } yield Ok(agglomerateGraph.toByteArray).as(protobufMimeType)
     }
   }
@@ -335,16 +333,14 @@ class DataSourceController @Inject()(
       UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
       for {
         agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-        largestAgglomerateId: Long <- agglomerateService
-          .largestAgglomerateId(
-            AgglomerateFileKey(
-              organizationId,
-              datasetDirectoryName,
-              dataLayerName,
-              mappingName
-            )
+        largestAgglomerateId: Long <- agglomerateService.largestAgglomerateId(
+          AgglomerateFileKey(
+            organizationId,
+            datasetDirectoryName,
+            dataLayerName,
+            mappingName
           )
-          .toFox
+        )
       } yield Ok(Json.toJson(largestAgglomerateId))
     }
   }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
index c4b5309caab..dfa848800f4 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
@@ -190,32 +190,35 @@ class DatasetArray(vaultPath: VaultPath,
   }
 
   def readAsMultiArray(shape: Array[Int], offset: Array[Long])(implicit ec: ExecutionContext,
-                                                               tc: TokenContext): Fox[MultiArray] = {
-    val totalOffset: Array[Long] = offset.zip(header.voxelOffset).map { case (o, v) => o - v }.padTo(offset.length, 0)
-    val chunkIndices = ChunkUtils.computeChunkIndices(datasetShape, chunkShape, shape, totalOffset)
-    if (partialCopyingIsNotNeededForMultiArray(shape, totalOffset, chunkIndices)) {
-      for {
-        chunkIndex <- chunkIndices.headOption.toFox
-        sourceChunk: MultiArray <- getSourceChunkDataWithCache(chunkIndex, useSkipTypingShortcut = true)
-      } yield sourceChunk
+                                                               tc: TokenContext): Fox[MultiArray] =
+    if (shape.product == 0) {
+      Fox.successful(MultiArrayUtils.createEmpty(rank))
     } else {
-      val targetBuffer = MultiArrayUtils.createDataBuffer(header.resolvedDataType, shape)
-      val targetMultiArray = MultiArrayUtils.createArrayWithGivenStorage(targetBuffer, shape)
-      val copiedFuture = Fox.combined(chunkIndices.map { chunkIndex: Array[Int] =>
+      val totalOffset: Array[Long] = offset.zip(header.voxelOffset).map { case (o, v) => o - v }.padTo(offset.length, 0)
+      val chunkIndices = ChunkUtils.computeChunkIndices(datasetShape, chunkShape, shape, totalOffset)
+      if (partialCopyingIsNotNeededForMultiArray(shape, totalOffset, chunkIndices)) {
         for {
-          sourceChunk: MultiArray <- getSourceChunkDataWithCache(chunkIndex)
-          offsetInChunk = computeOffsetInChunkIgnoringAxisOrder(chunkIndex, totalOffset)
-          _ <- tryo(MultiArrayUtils.copyRange(offsetInChunk, sourceChunk, targetMultiArray)).toFox ?~> formatCopyRangeErrorWithoutAxisOrder(
-            offsetInChunk,
-            sourceChunk,
-            targetMultiArray)
-        } yield ()
-      })
-      for {
-        _ <- copiedFuture
-      } yield targetMultiArray
+          chunkIndex <- chunkIndices.headOption.toFox
+          sourceChunk: MultiArray <- getSourceChunkDataWithCache(chunkIndex, useSkipTypingShortcut = true)
+        } yield sourceChunk
+      } else {
+        val targetBuffer = MultiArrayUtils.createDataBuffer(header.resolvedDataType, shape)
+        val targetMultiArray = MultiArrayUtils.createArrayWithGivenStorage(targetBuffer, shape)
+        val copiedFuture = Fox.combined(chunkIndices.map { chunkIndex: Array[Int] =>
+          for {
+            sourceChunk: MultiArray <- getSourceChunkDataWithCache(chunkIndex)
+            offsetInChunk = computeOffsetInChunkIgnoringAxisOrder(chunkIndex, totalOffset)
+            _ <- tryo(MultiArrayUtils.copyRange(offsetInChunk, sourceChunk, targetMultiArray)).toFox ?~> formatCopyRangeErrorWithoutAxisOrder(
+              offsetInChunk,
+              sourceChunk,
+              targetMultiArray)
+          } yield ()
+        })
+        for {
+          _ <- copiedFuture
+        } yield targetMultiArray
+      }
     }
-  }
 
   private def formatCopyRangeError(offsetInChunk: Array[Int], sourceChunk: MultiArray, target: MultiArray): String =
     s"Copying data from dataset chunk failed. Chunk shape (F): ${printAsOuterF(sourceChunk.getShape)}, target shape (F): ${printAsOuterF(
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
index f1af69b890c..a9a2160b7a6 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
@@ -45,6 +45,10 @@ object MultiArrayUtils extends LazyLogging {
     }
   }
 
+  def createEmpty(rank: Int): MultiArray = {
+    MultiArray.factory(MADataType.FLOAT, Array.fill(rank)(0))
+  }
+
   /**
     * Offset describes the displacement between source and target array.<br/>
     * <br/>
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 88d497c7fb3..88a76e099f0 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -150,37 +150,24 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
                        agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[SkeletonTracing] =
     for {
       before <- Instant.nowFox
-      agglomerate_to_segments_offsets <- openZarrArrayCached("agglomerate_to_segments_offsets")
-      agglomerate_to_edges_offsets <- openZarrArrayCached("agglomerate_to_edges_offsets")
+      agglomerateToSegmentsOffsets <- openZarrArrayCached("agglomerate_to_segments_offsets")
+      agglomerateToEdgesOffsets <- openZarrArrayCached("agglomerate_to_edges_offsets")
 
-      positionsRange: MultiArray <- agglomerate_to_segments_offsets.readAsMultiArray(shape = Array(2),
-                                                                                     offset = Array(agglomerateId))
-      edgesRange: MultiArray <- agglomerate_to_edges_offsets.readAsMultiArray(shape = Array(2),
-                                                                              offset = Array(agglomerateId))
+      positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(shape = Array(2),
+                                                                                  offset = Array(agglomerateId))
+      edgesRange: MultiArray <- agglomerateToEdgesOffsets.readAsMultiArray(shape = Array(2),
+                                                                           offset = Array(agglomerateId))
       nodeCount = positionsRange.getLong(1) - positionsRange.getLong(0)
       edgeCount = edgesRange.getLong(1) - edgesRange.getLong(0)
       edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
       _ <- Fox.fromBool(nodeCount <= edgeLimit) ?~> s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)"
       _ <- Fox.fromBool(edgeCount <= edgeLimit) ?~> s"Agglomerate has too many edges ($edgeCount > $edgeLimit)"
-      positions: MultiArray <- if (nodeCount == 0L) {
-        Fox.successful(MultiArray.factory(DataType.LONG, Array(0, 0)))
-      } else {
-        for {
-          agglomerate_to_positions <- openZarrArrayCached("agglomerate_to_positions")
-          positions <- agglomerate_to_positions.readAsMultiArray(offset = Array(positionsRange.getLong(0), 0),
-                                                                 shape = Array(nodeCount.toInt, 3))
-        } yield positions
-      }
-      edges: MultiArray <- if (edgeCount == 0L) {
-        Fox.successful(MultiArray.factory(DataType.LONG, Array(0, 0)))
-      } else {
-        for {
-          agglomerate_to_edges <- openZarrArrayCached("agglomerate_to_edges")
-          edges <- agglomerate_to_edges.readAsMultiArray(offset = Array(edgesRange.getLong(0), 0),
-                                                         shape = Array(edgeCount.toInt, 2))
-        } yield edges
-      }
-
+      agglomerateToPositions <- openZarrArrayCached("agglomerate_to_positions")
+      positions <- agglomerateToPositions.readAsMultiArray(offset = Array(positionsRange.getLong(0), 0),
+                                                           shape = Array(nodeCount.toInt, 3))
+      agglomerateToEdges <- openZarrArrayCached("agglomerate_to_edges")
+      edges: MultiArray <- agglomerateToEdges.readAsMultiArray(offset = Array(edgesRange.getLong(0), 0),
+                                                               shape = Array(edgeCount.toInt, 2))
       nodeIdStartAtOneOffset = 1
 
       // TODO use multiarray index iterators?
@@ -235,6 +222,69 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       shapeFirstElement <- tryo(shape(0)).toFox
     } yield shapeFirstElement
 
+  def generateAgglomerateGraph(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long)(
+      implicit ec: ExecutionContext,
+      tc: TokenContext): Fox[AgglomerateGraph] =
+    for {
+      agglomerateToSegmentsOffsets <- openZarrArrayCached("agglomerate_to_segments_offsets")
+      agglomerateToEdgesOffsets <- openZarrArrayCached("agglomerate_to_edges_offsets")
+
+      positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(shape = Array(2),
+                                                                                  offset = Array(agglomerateId))
+      edgesRange: MultiArray <- agglomerateToEdgesOffsets.readAsMultiArray(shape = Array(2),
+                                                                           offset = Array(agglomerateId))
+      nodeCount = positionsRange.getLong(1) - positionsRange.getLong(0)
+      edgeCount = edgesRange.getLong(1) - edgesRange.getLong(0)
+      edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
+      _ <- Fox.fromBool(nodeCount <= edgeLimit) ?~> s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)"
+      _ <- Fox.fromBool(edgeCount <= edgeLimit) ?~> s"Agglomerate has too many edges ($edgeCount > $edgeLimit)"
+      agglomerateToPositions <- openZarrArrayCached("agglomerate_to_positions")
+      positions: MultiArray <- agglomerateToPositions.readAsMultiArray(offset = Array(positionsRange.getLong(0), 0),
+                                                                       shape = Array(nodeCount.toInt, 3))
+      agglomerateToSegments <- openZarrArrayCached("agglomerate_to_segments")
+      segmentIds: MultiArray <- agglomerateToSegments.readAsMultiArray(offset = Array(positionsRange.getInt(0)),
+                                                                       shape = Array(nodeCount.toInt))
+      agglomerateToEdges <- openZarrArrayCached("agglomerate_to_edges")
+      edges: MultiArray <- agglomerateToEdges.readAsMultiArray(offset = Array(edgesRange.getLong(0), 0),
+                                                               shape = Array(edgeCount.toInt, 2))
+      agglomerateToAffinities <- openZarrArray("agglomerate_to_affinities")
+      affinities: MultiArray <- agglomerateToAffinities.readAsMultiArray(offset = Array(edgesRange.getLong(0)),
+                                                                         shape = Array(edgeCount.toInt))
+
+      agglomerateGraph = AgglomerateGraph(
+        // unsafeWrapArray is fine, because the underlying arrays are never mutated
+        segments = ArraySeq.unsafeWrapArray(segmentIds.getStorage.asInstanceOf[Array[Long]]),
+        edges = (0 until edges.getShape()(0)).map { edgeIdx: Int =>
+          AgglomerateEdge(
+            source = segmentIds.getLong(edges.getInt(edges.getIndex.set(Array(edgeIdx, 0)))),
+            target = segmentIds.getLong(edges.getInt(edges.getIndex.set(Array(edgeIdx, 1))))
+          )
+        },
+        positions = (0 until nodeCount.toInt).map { nodeIdx: Int =>
+          Vec3IntProto(
+            positions.getInt(positions.getIndex.set(Array(nodeIdx, 0))),
+            positions.getInt(positions.getIndex.set(Array(nodeIdx, 1))),
+            positions.getInt(positions.getIndex.set(Array(nodeIdx, 2)))
+          )
+        },
+        affinities = ArraySeq.unsafeWrapArray(affinities.getStorage.asInstanceOf[Array[Float]])
+      )
+    } yield agglomerateGraph
+
+  def segmentIdsForAgglomerateId(agglomerateFileKey: AgglomerateFileKey,
+                                 agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Seq[Long]] =
+    for {
+      agglomerateToSegmentsOffsets <- openZarrArrayCached("agglomerate_to_segments_offsets")
+      agglomerateToSegments <- openZarrArrayCached("agglomerate_to_segments")
+      segmentRange <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = Array(agglomerateId), shape = Array(2))
+      segmentCount = segmentRange.getLong(1) - segmentRange.getLong(0)
+      segmentIds <- if (segmentCount == 0)
+        Fox.successful(MultiArray.factory(DataType.LONG, Array(0, 0)))
+      else
+        agglomerateToSegments.readAsMultiArray(offset = Array(segmentRange.getLong(0)),
+                                               shape = Array(segmentCount.toInt))
+    } yield segmentIds.getStorage.asInstanceOf[Array[Long]].toSeq
+
 }
 
 class Hdf5AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
@@ -251,6 +301,9 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
   private val dataBaseDir = Paths.get(config.Datastore.baseDirectory)
   private val cumsumFileName = "cumsum.json"
 
+  // TODO remove
+  private val useZarr = true
+
   lazy val agglomerateFileCache = new AgglomerateFileCache(config.Datastore.Cache.AgglomerateFile.maxFileHandleEntries)
 
   def exploreAgglomerates(organizationId: String, datasetDirectoryName: String, dataLayerName: String): Set[String] = {
@@ -269,7 +322,7 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
 
   def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(implicit ec: ExecutionContext,
                                                                            tc: TokenContext): Fox[Array[Byte]] =
-    if (true) {
+    if (useZarr) {
       zarrAgglomerateService.applyAgglomerate(request)(data)
     } else applyAgglomerateHdf5(request)(data).toFox
 
@@ -368,7 +421,7 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
                        dataLayerName: String,
                        mappingName: String,
                        agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[SkeletonTracing] =
-    if (true) {
+    if (useZarr) {
       zarrAgglomerateService.generateSkeleton(organizationId,
                                               datasetDirectoryName,
                                               dataLayerName,
@@ -460,39 +513,45 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
       }).toFox
     }
 
-  def largestAgglomerateId(agglomerateFileKey: AgglomerateFileKey): Box[Long] = {
-    val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
-
-    tryo {
-      val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
-      reader.`object`().getNumberOfElements("/agglomerate_to_segments_offsets") - 1L
+  def largestAgglomerateId(agglomerateFileKey: AgglomerateFileKey)(implicit ec: ExecutionContext,
+                                                                   tc: TokenContext): Fox[Long] =
+    if (useZarr) zarrAgglomerateService.largestAgglomerateId(agglomerateFileKey)
+    else {
+      val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
+      tryo {
+        val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+        reader.`object`().getNumberOfElements("/agglomerate_to_segments_offsets") - 1L
+      }.toFox
     }
-  }
 
-  def segmentIdsForAgglomerateId(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long): Box[List[Long]] = {
-    val hdfFile =
-      dataBaseDir
-        .resolve(agglomerateFileKey.organizationId)
-        .resolve(agglomerateFileKey.datasetDirectoryName)
-        .resolve(agglomerateFileKey.layerName)
-        .resolve(agglomerateDir)
-        .resolve(s"${agglomerateFileKey.mappingName}.$agglomerateFileExtension")
-        .toFile
+  def segmentIdsForAgglomerateId(agglomerateFileKey: AgglomerateFileKey,
+                                 agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Seq[Long]] =
+    if (useZarr)
+      zarrAgglomerateService.segmentIdsForAgglomerateId(agglomerateFileKey, agglomerateId)
+    else {
+      val hdfFile =
+        dataBaseDir
+          .resolve(agglomerateFileKey.organizationId)
+          .resolve(agglomerateFileKey.datasetDirectoryName)
+          .resolve(agglomerateFileKey.layerName)
+          .resolve(agglomerateDir)
+          .resolve(s"${agglomerateFileKey.mappingName}.$agglomerateFileExtension")
+          .toFile
+
+      tryo {
+        val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+        val positionsRange: Array[Long] =
+          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
 
-    tryo {
-      val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
-      val positionsRange: Array[Long] =
-        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
-
-      val segmentCount = positionsRange(1) - positionsRange(0)
-      val segmentIds: Array[Long] =
-        if (segmentCount == 0) Array.empty[Long]
-        else {
-          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", segmentCount.toInt, positionsRange(0))
-        }
-      segmentIds.toList
+        val segmentCount = positionsRange(1) - positionsRange(0)
+        val segmentIds: Array[Long] =
+          if (segmentCount == 0) Array.empty[Long]
+          else {
+            reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", segmentCount.toInt, positionsRange(0))
+          }
+        segmentIds.toSeq
+      }.toFox
     }
-  }
 
   def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: Seq[Long]): Box[Seq[Long]] = {
     val cachedAgglomerateFile = agglomerateFileCache.withCache(agglomerateFileKey)(initHDFReader)
@@ -537,54 +596,60 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
       else binarySearchForSegment(rangeStart, middle - 1L, segmentId, reader)
     }
 
-  def generateAgglomerateGraph(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long): Box[AgglomerateGraph] =
-    tryo {
-      val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
+  def generateAgglomerateGraph(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long)(
+      implicit ec: ExecutionContext,
+      tc: TokenContext): Fox[AgglomerateGraph] =
+    if (useZarr)
+      zarrAgglomerateService.generateAgglomerateGraph(agglomerateFileKey, agglomerateId)
+    else {
+      tryo {
+        val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
 
-      val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+        val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
 
-      val positionsRange: Array[Long] =
-        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
-      val edgesRange: Array[Long] =
-        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
+        val positionsRange: Array[Long] =
+          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
+        val edgesRange: Array[Long] =
+          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
 
-      val nodeCount = positionsRange(1) - positionsRange(0)
-      val edgeCount = edgesRange(1) - edgesRange(0)
-      val edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
-      if (nodeCount > edgeLimit) {
-        throw new Exception(s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)")
-      }
-      if (edgeCount > edgeLimit) {
-        throw new Exception(s"Agglomerate has too many edges ($edgeCount > $edgeLimit)")
-      }
-      val segmentIds: Array[Long] =
-        if (nodeCount == 0L) Array[Long]()
-        else
-          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", nodeCount.toInt, positionsRange(0))
-      val positions: Array[Array[Long]] =
-        if (nodeCount == 0L) Array[Array[Long]]()
-        else
-          reader
-            .uint64()
-            .readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
-      val edges: Array[Array[Long]] =
-        if (edgeCount == 0L) Array[Array[Long]]()
-        else
-          reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
-      val affinities: Array[Float] =
-        if (edgeCount == 0L) Array[Float]()
-        else
-          reader.float32().readArrayBlockWithOffset("/agglomerate_to_affinities", edgeCount.toInt, edgesRange(0))
-
-      AgglomerateGraph(
-        // unsafeWrapArray is fine, because the underlying arrays are never mutated
-        segments = ArraySeq.unsafeWrapArray(segmentIds),
-        edges = ArraySeq.unsafeWrapArray(
-          edges.map(e => AgglomerateEdge(source = segmentIds(e(0).toInt), target = segmentIds(e(1).toInt)))),
-        positions =
-          ArraySeq.unsafeWrapArray(positions.map(pos => Vec3IntProto(pos(0).toInt, pos(1).toInt, pos(2).toInt))),
-        affinities = ArraySeq.unsafeWrapArray(affinities)
-      )
+        val nodeCount = positionsRange(1) - positionsRange(0)
+        val edgeCount = edgesRange(1) - edgesRange(0)
+        val edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
+        if (nodeCount > edgeLimit) {
+          throw new Exception(s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)")
+        }
+        if (edgeCount > edgeLimit) {
+          throw new Exception(s"Agglomerate has too many edges ($edgeCount > $edgeLimit)")
+        }
+        val segmentIds: Array[Long] =
+          if (nodeCount == 0L) Array[Long]()
+          else
+            reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", nodeCount.toInt, positionsRange(0))
+        val positions: Array[Array[Long]] =
+          if (nodeCount == 0L) Array[Array[Long]]()
+          else
+            reader
+              .uint64()
+              .readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
+        val edges: Array[Array[Long]] =
+          if (edgeCount == 0L) Array[Array[Long]]()
+          else
+            reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
+        val affinities: Array[Float] =
+          if (edgeCount == 0L) Array[Float]()
+          else
+            reader.float32().readArrayBlockWithOffset("/agglomerate_to_affinities", edgeCount.toInt, edgesRange(0))
+
+        AgglomerateGraph(
+          // unsafeWrapArray is fine, because the underlying arrays are never mutated
+          segments = ArraySeq.unsafeWrapArray(segmentIds),
+          edges = ArraySeq.unsafeWrapArray(
+            edges.map(e => AgglomerateEdge(source = segmentIds(e(0).toInt), target = segmentIds(e(1).toInt)))),
+          positions =
+            ArraySeq.unsafeWrapArray(positions.map(pos => Vec3IntProto(pos(0).toInt, pos(1).toInt, pos(2).toInt))),
+          affinities = ArraySeq.unsafeWrapArray(affinities)
+        )
+      }.toFox
     }
 
 }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala
index 541b5a040d5..108d6b4c9ce 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala
@@ -185,11 +185,11 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig,
         ))
       bucketData <- binaryDataServiceHolder.binaryDataService.handleMultipleBucketRequests(bucketRequests)
     } yield (bucketData, dataLayer.elementClass)
-  private def getBucketPositions(
-      organizationId: String,
-      datasetDirectoryName: String,
-      dataLayerName: String,
-      mappingName: Option[String])(segmentOrAgglomerateId: Long, mag: Vec3Int): Fox[Set[Vec3IntProto]] =
+  private def getBucketPositions(organizationId: String,
+                                 datasetDirectoryName: String,
+                                 dataLayerName: String,
+                                 mappingName: Option[String])(segmentOrAgglomerateId: Long, mag: Vec3Int)(
+      implicit tc: TokenContext): Fox[Set[Vec3IntProto]] =
     for {
       segmentIds <- getSegmentIdsForAgglomerateIdIfNeeded(organizationId,
                                                           datasetDirectoryName,
@@ -212,11 +212,12 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig,
       bucketPositions = bucketPositionsInFileMag.map(_ / (mag / fileMag))
     } yield bucketPositions
 
-  private def getSegmentIdsForAgglomerateIdIfNeeded(organizationId: String,
-                                                    datasetDirectoryName: String,
-                                                    dataLayerName: String,
-                                                    segmentOrAgglomerateId: Long,
-                                                    mappingNameOpt: Option[String]): Fox[List[Long]] =
+  private def getSegmentIdsForAgglomerateIdIfNeeded(
+      organizationId: String,
+      datasetDirectoryName: String,
+      dataLayerName: String,
+      segmentOrAgglomerateId: Long,
+      mappingNameOpt: Option[String])(implicit tc: TokenContext): Fox[Seq[Long]] =
     // Editable mappings cannot happen here since those requests go to the tracingstore
     mappingNameOpt match {
       case Some(mappingName) =>
@@ -228,14 +229,12 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig,
             dataLayerName,
             mappingName
           )
-          largestAgglomerateId <- agglomerateService.largestAgglomerateId(agglomerateFileKey).toFox
+          largestAgglomerateId <- agglomerateService.largestAgglomerateId(agglomerateFileKey)
           segmentIds <- if (segmentOrAgglomerateId <= largestAgglomerateId) {
-            agglomerateService
-              .segmentIdsForAgglomerateId(
-                agglomerateFileKey,
-                segmentOrAgglomerateId
-              )
-              .toFox
+            agglomerateService.segmentIdsForAgglomerateId(
+              agglomerateFileKey,
+              segmentOrAgglomerateId
+            )
           } else
             Fox.successful(List.empty) // agglomerate id is outside of file range, was likely created during brushing
         } yield segmentIds
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshFileService.scala
index ac577ef2dbf..92f04699283 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshFileService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshFileService.scala
@@ -163,12 +163,11 @@ class MeshFileService @Inject()(config: DataStoreConfig)(implicit ec: ExecutionC
       .toOption
       .getOrElse(0)
 
-  def listMeshChunksForSegmentsMerged(
-      organizationId: String,
-      datasetDirectoryName: String,
-      dataLayerName: String,
-      meshFileName: String,
-      segmentIds: List[Long])(implicit m: MessagesProvider): Fox[WebknossosSegmentInfo] =
+  def listMeshChunksForSegmentsMerged(organizationId: String,
+                                      datasetDirectoryName: String,
+                                      dataLayerName: String,
+                                      meshFileName: String,
+                                      segmentIds: Seq[Long])(implicit m: MessagesProvider): Fox[WebknossosSegmentInfo] =
     for {
       _ <- Fox.successful(())
       meshFilePath: Path = dataBaseDir
@@ -190,12 +189,12 @@ class MeshFileService @Inject()(config: DataStoreConfig)(implicit ec: ExecutionC
     } yield wkChunkInfos
 
   private def listMeshChunksForSegments(meshFilePath: Path,
-                                        segmentIds: List[Long],
+                                        segmentIds: Seq[Long],
                                         lodScaleMultiplier: Double,
                                         transform: Array[Array[Double]]): List[List[MeshLodInfo]] =
     meshFileCache
       .withCachedHdf5(meshFilePath) { cachedMeshFile: CachedHdf5File =>
-        segmentIds.flatMap(segmentId =>
+        segmentIds.toList.flatMap(segmentId =>
           listMeshChunksForSegment(cachedMeshFile, segmentId, lodScaleMultiplier, transform))
       }
       .toOption
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshMappingHelper.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshMappingHelper.scala
index ecee0011d7a..96a688e980c 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshMappingHelper.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshMappingHelper.scala
@@ -27,7 +27,7 @@ trait MeshMappingHelper extends FoxImplicits {
       agglomerateId: Long,
       mappingNameForMeshFile: Option[String],
       omitMissing: Boolean // If true, failing lookups in the agglomerate file will just return empty list.
-  )(implicit ec: ExecutionContext, tc: TokenContext): Fox[List[Long]] =
+  )(implicit ec: ExecutionContext, tc: TokenContext): Fox[Seq[Long]] =
     (targetMappingName, editableMappingTracingId) match {
       case (None, None) =>
         // No mapping selected, assume id matches meshfile
@@ -40,15 +40,17 @@ trait MeshMappingHelper extends FoxImplicits {
         // assume agglomerate id, fetch oversegmentation segment ids for it
         for {
           agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-          segmentIdsBox = agglomerateService.segmentIdsForAgglomerateId(
-            AgglomerateFileKey(
-              organizationId,
-              datasetDirectoryName,
-              dataLayerName,
-              mappingName
-            ),
-            agglomerateId
-          )
+          segmentIdsBox <- agglomerateService
+            .segmentIdsForAgglomerateId(
+              AgglomerateFileKey(
+                organizationId,
+                datasetDirectoryName,
+                dataLayerName,
+                mappingName
+              ),
+              agglomerateId
+            )
+            .shiftBox
           segmentIds <- segmentIdsBox match {
             case Full(segmentIds) => Fox.successful(segmentIds)
             case _                => if (omitMissing) Fox.successful(List.empty) else segmentIdsBox.toFox
@@ -67,17 +69,15 @@ trait MeshMappingHelper extends FoxImplicits {
           else // the agglomerate id is not present in the editable mapping. Fetch its info from the base mapping.
             for {
               agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-              localSegmentIds <- agglomerateService
-                .segmentIdsForAgglomerateId(
-                  AgglomerateFileKey(
-                    organizationId,
-                    datasetDirectoryName,
-                    dataLayerName,
-                    mappingName
-                  ),
-                  agglomerateId
-                )
-                .toFox
+              localSegmentIds <- agglomerateService.segmentIdsForAgglomerateId(
+                AgglomerateFileKey(
+                  organizationId,
+                  datasetDirectoryName,
+                  dataLayerName,
+                  mappingName
+                ),
+                agglomerateId
+              )
             } yield localSegmentIds
         } yield segmentIds
       case _ => Fox.failure("Cannot determine segment ids for editable mapping without base mapping")
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/NeuroglancerPrecomputedMeshFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/NeuroglancerPrecomputedMeshFileService.scala
index 91bfdc51bc1..0497dca3b87 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/NeuroglancerPrecomputedMeshFileService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/NeuroglancerPrecomputedMeshFileService.scala
@@ -125,7 +125,7 @@ class NeuroglancerPrecomputedMeshFileService @Inject()(config: DataStoreConfig,
     )
   }
 
-  def listMeshChunksForMultipleSegments(meshFilePathOpt: Option[String], segmentId: List[Long])(
+  def listMeshChunksForMultipleSegments(meshFilePathOpt: Option[String], segmentId: Seq[Long])(
       implicit tc: TokenContext): Fox[WebknossosSegmentInfo] =
     for {
       meshFilePath <- meshFilePathOpt.toFox ?~> "No mesh file path provided"

From 0439bcee0b2c871bacd02b720a6cc75bd187115f Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 27 May 2025 16:16:56 +0200
Subject: [PATCH 14/22] remove unused test

---
 .../datastore/controllers/Application.scala       |  8 --------
 .../datastore/services/AgglomerateService.scala   | 15 ++++-----------
 .../com.scalableminds.webknossos.datastore.routes |  1 -
 3 files changed, 4 insertions(+), 20 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
index f22b572a55d..7a3802c5e64 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
@@ -40,14 +40,6 @@ class Application @Inject()(redisClient: DataStoreRedisStore,
     }
   }
 
-  def testAgglomerateZarr: Action[AnyContent] = Action.async { implicit request =>
-    log() {
-      for {
-        data <- agglomerateService.readFromSegmentToAgglomerate
-      } yield Ok(s"got ${data.getSize} elements of type ${data.getDataType}: ${data.toString}")
-    }
-  }
-
   // Test that the NativeBucketScanner works.
   // The result is stored in a val because we expect that this continues to work if it works on startup.
   private lazy val testNativeBucketScanner = tryo {
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 88a76e099f0..37a7f20dfe6 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -59,13 +59,6 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
 
   protected lazy val bucketScanner = new NativeBucketScanner()
 
-  def readFromSegmentToAgglomerate(implicit ec: ExecutionContext, tc: TokenContext): Fox[ucar.ma2.Array] =
-    for {
-      zarrArray <- openZarrArrayCached("segment_to_agglomerate")
-      read <- zarrArray.readAsMultiArray(Array(10), Array(2))
-      _ = logger.info(s"read ${read.getSize} elements from agglomerate file segmentToAgglomerate")
-    } yield read
-
   private def mapSingleSegment(zarrArray: DatasetArray, segmentId: Long)(implicit ec: ExecutionContext,
                                                                          tc: TokenContext): Fox[Long] =
     for {
@@ -153,10 +146,10 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       agglomerateToSegmentsOffsets <- openZarrArrayCached("agglomerate_to_segments_offsets")
       agglomerateToEdgesOffsets <- openZarrArrayCached("agglomerate_to_edges_offsets")
 
-      positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(shape = Array(2),
-                                                                                  offset = Array(agglomerateId))
-      edgesRange: MultiArray <- agglomerateToEdgesOffsets.readAsMultiArray(shape = Array(2),
-                                                                           offset = Array(agglomerateId))
+      positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = Array(agglomerateId),
+                                                                                  shape = Array(2))
+      edgesRange: MultiArray <- agglomerateToEdgesOffsets.readAsMultiArray(offset = Array(agglomerateId),
+                                                                           shape = Array(2))
       nodeCount = positionsRange.getLong(1) - positionsRange.getLong(0)
       edgeCount = edgesRange.getLong(1) - edgesRange.getLong(0)
       edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
index bcaf26dc3f6..fef187d16e3 100644
--- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
+++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
@@ -3,7 +3,6 @@
 
 # Health endpoint
 GET           /health                                                                                                                                 @com.scalableminds.webknossos.datastore.controllers.Application.health
-GET           /testAgglomerateZarr                                                                                                                    @com.scalableminds.webknossos.datastore.controllers.Application.testAgglomerateZarr
 
 # Read image data
 POST          /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/data                                                              @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaWebknossos(organizationId: String, datasetDirectoryName: String, dataLayerName: String)

From 11027d717554896b920166c7c24f36057aa35cab Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 28 May 2025 10:02:59 +0200
Subject: [PATCH 15/22] implement positionForSegmentId;
 agglomerateIdsForSegmentIds

---
 .../datastore/DataStoreModule.scala           |   1 +
 .../controllers/DataSourceController.scala    |  27 ++---
 .../services/AgglomerateService.scala         | 114 +++++++++++++-----
 3 files changed, 96 insertions(+), 46 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala
index 7a2bd9b28ba..4b1ee3c06a2 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala
@@ -28,6 +28,7 @@ class DataStoreModule extends AbstractModule {
     bind(classOf[BinaryDataServiceHolder]).asEagerSingleton()
     bind(classOf[MappingService]).asEagerSingleton()
     bind(classOf[AgglomerateService]).asEagerSingleton()
+    bind(classOf[ZarrAgglomerateService]).asEagerSingleton()
     bind(classOf[AdHocMeshServiceHolder]).asEagerSingleton()
     bind(classOf[ApplicationHealthService]).asEagerSingleton()
     bind(classOf[DSDatasetErrorLoggingService]).asEagerSingleton()
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
index 06508d7c469..a8ae4237f45 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
@@ -315,10 +315,9 @@ class DataSourceController @Inject()(
       UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
       for {
         agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-        position <- agglomerateService
-          .positionForSegmentId(AgglomerateFileKey(organizationId, datasetDirectoryName, dataLayerName, mappingName),
-                                segmentId)
-          .toFox ?~> "getSegmentPositionFromAgglomerateFile.failed"
+        position <- agglomerateService.positionForSegmentId(
+          AgglomerateFileKey(organizationId, datasetDirectoryName, dataLayerName, mappingName),
+          segmentId) ?~> "getSegmentPositionFromAgglomerateFile.failed"
       } yield Ok(Json.toJson(position))
     }
   }
@@ -355,17 +354,15 @@ class DataSourceController @Inject()(
       UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
       for {
         agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-        agglomerateIds: Seq[Long] <- agglomerateService
-          .agglomerateIdsForSegmentIds(
-            AgglomerateFileKey(
-              organizationId,
-              datasetDirectoryName,
-              dataLayerName,
-              mappingName
-            ),
-            request.body.items
-          )
-          .toFox
+        agglomerateIds: Seq[Long] <- agglomerateService.agglomerateIdsForSegmentIds(
+          AgglomerateFileKey(
+            organizationId,
+            datasetDirectoryName,
+            dataLayerName,
+            mappingName
+          ),
+          request.body.items
+        )
       } yield Ok(ListOfLong(agglomerateIds).toByteArray)
     }
   }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 37a7f20dfe6..2228c8b183f 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -59,10 +59,10 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
 
   protected lazy val bucketScanner = new NativeBucketScanner()
 
-  private def mapSingleSegment(zarrArray: DatasetArray, segmentId: Long)(implicit ec: ExecutionContext,
-                                                                         tc: TokenContext): Fox[Long] =
+  private def mapSingleSegment(segmentToAgglomerate: DatasetArray, segmentId: Long)(implicit ec: ExecutionContext,
+                                                                                    tc: TokenContext): Fox[Long] =
     for {
-      asMultiArray <- zarrArray.readAsMultiArray(shape = Array(1), offset = Array(segmentId))
+      asMultiArray <- segmentToAgglomerate.readAsMultiArray(shape = Array(1), offset = Array(segmentId))
     } yield asMultiArray.getLong(0)
 
   private def openZarrArrayCached(zarrArrayName: String)(implicit ec: ExecutionContext, tc: TokenContext) =
@@ -109,11 +109,11 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       bucketScanner.collectSegmentIds(data, bytesPerElement, isSigned = false, skipZeroes = false)
 
     for {
-      zarrArray <- openZarrArrayCached("segment_to_agglomerate")
+      segmentToAgglomerate <- openZarrArrayCached("segment_to_agglomerate")
       beforeBuildMap = Instant.now
       relevantAgglomerateMap: Map[Long, Long] <- Fox
         .serialCombined(distinctSegmentIds) { segmentId =>
-          mapSingleSegment(zarrArray, segmentId).map((segmentId, _))
+          mapSingleSegment(segmentToAgglomerate, segmentId).map((segmentId, _))
         }
         .map(_.toMap)
       _ = Instant.logSince(beforeBuildMap, "build map")
@@ -278,6 +278,51 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
                                                shape = Array(segmentCount.toInt))
     } yield segmentIds.getStorage.asInstanceOf[Array[Long]].toSeq
 
+  def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: Seq[Long])(
+      implicit ec: ExecutionContext,
+      tc: TokenContext): Fox[Seq[Long]] =
+    for {
+      segmentToAgglomerate <- openZarrArrayCached("segment_to_agglomerate")
+      agglomerateIds <- Fox.serialCombined(segmentIds) { segmentId =>
+        mapSingleSegment(segmentToAgglomerate, segmentId)
+      }
+    } yield agglomerateIds
+
+  def positionForSegmentId(agglomerateFileKey: AgglomerateFileKey, segmentId: Long)(implicit ec: ExecutionContext,
+                                                                                    tc: TokenContext): Fox[Vec3Int] =
+    for {
+      segmentToAgglomerate <- openZarrArrayCached("segment_to_agglomerate")
+      agglomerateId <- mapSingleSegment(segmentToAgglomerate, segmentId)
+      agglomerateToSegmentsOffsets <- openZarrArrayCached("agglomerate_to_segments_offsets")
+      segmentsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = Array(agglomerateId),
+                                                                                 shape = Array(2))
+      agglomerateToSegments <- openZarrArrayCached("agglomerate_to_segments")
+      segmentIndex <- binarySearchForSegment(segmentsRange.getLong(0),
+                                             segmentsRange.getLong(1),
+                                             segmentId,
+                                             agglomerateToSegments)
+      agglomerateToPositions <- openZarrArrayCached("agglomerate_to_positions")
+      position <- agglomerateToPositions.readAsMultiArray(offset = Array(segmentIndex, 0), shape = Array(3, 1))
+    } yield Vec3Int(position.getInt(0), position.getInt(1), position.getInt(2))
+
+  private def binarySearchForSegment(
+      rangeStart: Long,
+      rangeEnd: Long,
+      segmentId: Long,
+      agglomerateToSegments: DatasetArray)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Long] =
+    if (rangeStart > rangeEnd) Fox.failure("Could not find segmentId in agglomerate file")
+    else {
+      val middle = rangeStart + (rangeEnd - rangeStart) / 2
+      for {
+        segmentIdAtMiddleMA <- agglomerateToSegments.readAsMultiArray(offset = Array(middle), shape = Array(1))
+        segmentIdAtMiddle = segmentIdAtMiddleMA.getLong(0)
+        segmentIndex <- if (segmentIdAtMiddle == segmentId)
+          Fox.successful(middle)
+        else if (segmentIdAtMiddle < segmentId) {
+          binarySearchForSegment(middle + 1L, rangeEnd, segmentId, agglomerateToSegments)
+        } else binarySearchForSegment(rangeStart, middle - 1L, segmentId, agglomerateToSegments)
+      } yield segmentIndex
+    }
 }
 
 class Hdf5AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
@@ -546,34 +591,41 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
       }.toFox
     }
 
-  def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: Seq[Long]): Box[Seq[Long]] = {
-    val cachedAgglomerateFile = agglomerateFileCache.withCache(agglomerateFileKey)(initHDFReader)
-
-    tryo {
-      val agglomerateIds = segmentIds.map { segmentId: Long =>
-        cachedAgglomerateFile.agglomerateIdCache.withCache(segmentId,
-                                                           cachedAgglomerateFile.reader,
-                                                           cachedAgglomerateFile.dataset)(readHDF)
-      }
-      cachedAgglomerateFile.finishAccess()
-      agglomerateIds
+  def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: Seq[Long])(
+      implicit ec: ExecutionContext,
+      tc: TokenContext): Fox[Seq[Long]] =
+    if (useZarr) {
+      zarrAgglomerateService.agglomerateIdsForSegmentIds(agglomerateFileKey, segmentIds)
+    } else {
+      val cachedAgglomerateFile = agglomerateFileCache.withCache(agglomerateFileKey)(initHDFReader)
+      tryo {
+        val agglomerateIds = segmentIds.map { segmentId: Long =>
+          cachedAgglomerateFile.agglomerateIdCache.withCache(segmentId,
+                                                             cachedAgglomerateFile.reader,
+                                                             cachedAgglomerateFile.dataset)(readHDF)
+        }
+        cachedAgglomerateFile.finishAccess()
+        agglomerateIds
+      }.toFox
     }
 
-  }
-
-  def positionForSegmentId(agglomerateFileKey: AgglomerateFileKey, segmentId: Long): Box[Vec3Int] = {
-    val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
-    val reader: IHDF5Reader = HDF5FactoryProvider.get.openForReading(hdfFile)
-    for {
-      agglomerateIdArr: Array[Long] <- tryo(
-        reader.uint64().readArrayBlockWithOffset("/segment_to_agglomerate", 1, segmentId))
-      agglomerateId = agglomerateIdArr(0)
-      segmentsRange: Array[Long] <- tryo(
-        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId))
-      segmentIndex <- binarySearchForSegment(segmentsRange(0), segmentsRange(1), segmentId, reader)
-      position <- tryo(reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_positions", 1, 3, segmentIndex, 0)(0))
-    } yield Vec3Int(position(0).toInt, position(1).toInt, position(2).toInt)
-  }
+  def positionForSegmentId(agglomerateFileKey: AgglomerateFileKey, segmentId: Long)(implicit ec: ExecutionContext,
+                                                                                    tc: TokenContext): Fox[Vec3Int] =
+    if (useZarr) zarrAgglomerateService.positionForSegmentId(agglomerateFileKey, segmentId)
+    else {
+      val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
+      val reader: IHDF5Reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+      (for {
+        agglomerateIdArr: Array[Long] <- tryo(
+          reader.uint64().readArrayBlockWithOffset("/segment_to_agglomerate", 1, segmentId))
+        agglomerateId = agglomerateIdArr(0)
+        segmentsRange: Array[Long] <- tryo(
+          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId))
+        segmentIndex <- binarySearchForSegment(segmentsRange(0), segmentsRange(1), segmentId, reader)
+        position <- tryo(
+          reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_positions", 1, 3, segmentIndex, 0)(0))
+      } yield Vec3Int(position(0).toInt, position(1).toInt, position(2).toInt)).toFox
+    }
 
   @tailrec
   private def binarySearchForSegment(rangeStart: Long,

From 0c5d6471e5426402c45864ea8548e4c63cc53c28 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 28 May 2025 10:15:14 +0200
Subject: [PATCH 16/22] select mapping by request

---
 .../services/AgglomerateService.scala         | 84 ++++++++++++-------
 1 file changed, 53 insertions(+), 31 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 2228c8b183f..94daad2d85a 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -39,7 +39,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
   private val agglomerateDir = "agglomerates"
 
   // TODO clear on dataset reload
-  private lazy val openArraysCache = AlfuCache[String, DatasetArray]()
+  private lazy val openArraysCache = AlfuCache[(AgglomerateFileKey, String), DatasetArray]()
 
   // TODO unify with existing chunkContentsCache from binaryDataService?
   private lazy val sharedChunkContentsCache: AlfuCache[String, MultiArray] = {
@@ -65,15 +65,16 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       asMultiArray <- segmentToAgglomerate.readAsMultiArray(shape = Array(1), offset = Array(segmentId))
     } yield asMultiArray.getLong(0)
 
-  private def openZarrArrayCached(zarrArrayName: String)(implicit ec: ExecutionContext, tc: TokenContext) =
-    openArraysCache.getOrLoad(zarrArrayName, zarrArrayName => openZarrArray(zarrArrayName))
+  private def openZarrArrayCached(agglomerateFileKey: AgglomerateFileKey,
+                                  zarrArrayName: String)(implicit ec: ExecutionContext, tc: TokenContext) =
+    openArraysCache.getOrLoad((agglomerateFileKey, zarrArrayName),
+                              _ => openZarrArray(agglomerateFileKey, zarrArrayName))
 
-  private def openZarrArray(zarrArrayName: String)(implicit ec: ExecutionContext,
-                                                   tc: TokenContext): Fox[DatasetArray] = {
-    val zarrGroupPath =
-      dataBaseDir
-        .resolve("sample_organization/test-agglomerate-file-zarr/segmentation/agglomerates/agglomerate_view_55")
-        .toAbsolutePath
+  private def openZarrArray(agglomerateFileKey: AgglomerateFileKey, zarrArrayName: String)(
+      implicit ec: ExecutionContext,
+      tc: TokenContext): Fox[DatasetArray] = {
+
+    val zarrGroupPath = agglomerateFileKey.zarrGroupPath(dataBaseDir, agglomerateDir).toAbsolutePath
     for {
       groupVaultPath <- dataVaultService.getVaultPath(RemoteSourceDescriptor(new URI(s"file://$zarrGroupPath"), None))
       segmentToAgglomeratePath = groupVaultPath / zarrArrayName
@@ -91,7 +92,6 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
                                                                            tc: TokenContext): Fox[Array[Byte]] = {
 
     val agglomerateFileKey = AgglomerateFileKey.fromDataRequest(request)
-    val zarrGroupPath = agglomerateFileKey.zarrGroupPath(dataBaseDir, agglomerateDir).toAbsolutePath
 
     def convertToAgglomerate(segmentIds: Array[Long],
                              relevantAgglomerateMap: Map[Long, Long],
@@ -109,7 +109,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       bucketScanner.collectSegmentIds(data, bytesPerElement, isSigned = false, skipZeroes = false)
 
     for {
-      segmentToAgglomerate <- openZarrArrayCached("segment_to_agglomerate")
+      segmentToAgglomerate <- openZarrArrayCached(agglomerateFileKey, "segment_to_agglomerate")
       beforeBuildMap = Instant.now
       relevantAgglomerateMap: Map[Long, Long] <- Fox
         .serialCombined(distinctSegmentIds) { segmentId =>
@@ -143,8 +143,9 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
                        agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[SkeletonTracing] =
     for {
       before <- Instant.nowFox
-      agglomerateToSegmentsOffsets <- openZarrArrayCached("agglomerate_to_segments_offsets")
-      agglomerateToEdgesOffsets <- openZarrArrayCached("agglomerate_to_edges_offsets")
+      agglomerateFileKey = AgglomerateFileKey(organizationId, datasetDirectoryName, dataLayerName, mappingName)
+      agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
+      agglomerateToEdgesOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges_offsets")
 
       positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = Array(agglomerateId),
                                                                                   shape = Array(2))
@@ -155,10 +156,10 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
       _ <- Fox.fromBool(nodeCount <= edgeLimit) ?~> s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)"
       _ <- Fox.fromBool(edgeCount <= edgeLimit) ?~> s"Agglomerate has too many edges ($edgeCount > $edgeLimit)"
-      agglomerateToPositions <- openZarrArrayCached("agglomerate_to_positions")
+      agglomerateToPositions <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_positions")
       positions <- agglomerateToPositions.readAsMultiArray(offset = Array(positionsRange.getLong(0), 0),
                                                            shape = Array(nodeCount.toInt, 3))
-      agglomerateToEdges <- openZarrArrayCached("agglomerate_to_edges")
+      agglomerateToEdges <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges")
       edges: MultiArray <- agglomerateToEdges.readAsMultiArray(offset = Array(edgesRange.getLong(0), 0),
                                                                shape = Array(edgeCount.toInt, 2))
       nodeIdStartAtOneOffset = 1
@@ -210,7 +211,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
   def largestAgglomerateId(agglomerateFileKey: AgglomerateFileKey)(implicit ec: ExecutionContext,
                                                                    tc: TokenContext): Fox[Long] =
     for {
-      array <- openZarrArrayCached("agglomerate_to_segments_offsets")
+      array <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
       shape <- array.datasetShape.toFox ?~> "Could not determine array shape"
       shapeFirstElement <- tryo(shape(0)).toFox
     } yield shapeFirstElement
@@ -219,8 +220,8 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       implicit ec: ExecutionContext,
       tc: TokenContext): Fox[AgglomerateGraph] =
     for {
-      agglomerateToSegmentsOffsets <- openZarrArrayCached("agglomerate_to_segments_offsets")
-      agglomerateToEdgesOffsets <- openZarrArrayCached("agglomerate_to_edges_offsets")
+      agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
+      agglomerateToEdgesOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges_offsets")
 
       positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(shape = Array(2),
                                                                                   offset = Array(agglomerateId))
@@ -231,16 +232,16 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
       _ <- Fox.fromBool(nodeCount <= edgeLimit) ?~> s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)"
       _ <- Fox.fromBool(edgeCount <= edgeLimit) ?~> s"Agglomerate has too many edges ($edgeCount > $edgeLimit)"
-      agglomerateToPositions <- openZarrArrayCached("agglomerate_to_positions")
+      agglomerateToPositions <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_positions")
       positions: MultiArray <- agglomerateToPositions.readAsMultiArray(offset = Array(positionsRange.getLong(0), 0),
                                                                        shape = Array(nodeCount.toInt, 3))
-      agglomerateToSegments <- openZarrArrayCached("agglomerate_to_segments")
+      agglomerateToSegments <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments")
       segmentIds: MultiArray <- agglomerateToSegments.readAsMultiArray(offset = Array(positionsRange.getInt(0)),
                                                                        shape = Array(nodeCount.toInt))
-      agglomerateToEdges <- openZarrArrayCached("agglomerate_to_edges")
+      agglomerateToEdges <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges")
       edges: MultiArray <- agglomerateToEdges.readAsMultiArray(offset = Array(edgesRange.getLong(0), 0),
                                                                shape = Array(edgeCount.toInt, 2))
-      agglomerateToAffinities <- openZarrArray("agglomerate_to_affinities")
+      agglomerateToAffinities <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_affinities")
       affinities: MultiArray <- agglomerateToAffinities.readAsMultiArray(offset = Array(edgesRange.getLong(0)),
                                                                          shape = Array(edgeCount.toInt))
 
@@ -267,8 +268,8 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
   def segmentIdsForAgglomerateId(agglomerateFileKey: AgglomerateFileKey,
                                  agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Seq[Long]] =
     for {
-      agglomerateToSegmentsOffsets <- openZarrArrayCached("agglomerate_to_segments_offsets")
-      agglomerateToSegments <- openZarrArrayCached("agglomerate_to_segments")
+      agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
+      agglomerateToSegments <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments")
       segmentRange <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = Array(agglomerateId), shape = Array(2))
       segmentCount = segmentRange.getLong(1) - segmentRange.getLong(0)
       segmentIds <- if (segmentCount == 0)
@@ -282,7 +283,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       implicit ec: ExecutionContext,
       tc: TokenContext): Fox[Seq[Long]] =
     for {
-      segmentToAgglomerate <- openZarrArrayCached("segment_to_agglomerate")
+      segmentToAgglomerate <- openZarrArrayCached(agglomerateFileKey, "segment_to_agglomerate")
       agglomerateIds <- Fox.serialCombined(segmentIds) { segmentId =>
         mapSingleSegment(segmentToAgglomerate, segmentId)
       }
@@ -291,17 +292,17 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
   def positionForSegmentId(agglomerateFileKey: AgglomerateFileKey, segmentId: Long)(implicit ec: ExecutionContext,
                                                                                     tc: TokenContext): Fox[Vec3Int] =
     for {
-      segmentToAgglomerate <- openZarrArrayCached("segment_to_agglomerate")
+      segmentToAgglomerate <- openZarrArrayCached(agglomerateFileKey, "segment_to_agglomerate")
       agglomerateId <- mapSingleSegment(segmentToAgglomerate, segmentId)
-      agglomerateToSegmentsOffsets <- openZarrArrayCached("agglomerate_to_segments_offsets")
+      agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
       segmentsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = Array(agglomerateId),
                                                                                  shape = Array(2))
-      agglomerateToSegments <- openZarrArrayCached("agglomerate_to_segments")
+      agglomerateToSegments <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments")
       segmentIndex <- binarySearchForSegment(segmentsRange.getLong(0),
                                              segmentsRange.getLong(1),
                                              segmentId,
                                              agglomerateToSegments)
-      agglomerateToPositions <- openZarrArrayCached("agglomerate_to_positions")
+      agglomerateToPositions <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_positions")
       position <- agglomerateToPositions.readAsMultiArray(offset = Array(segmentIndex, 0), shape = Array(3, 1))
     } yield Vec3Int(position.getInt(0), position.getInt(1), position.getInt(2))
 
@@ -354,8 +355,29 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
         paths.map(path => FilenameUtils.removeExtension(path.getFileName.toString))
       }
       .toOption
-      .getOrElse(Nil)
-      .toSet ++ Set("agglomerate_view_5") // TODO
+      .getOrElse(Nil) // TODO explore zarr agglomerates?
+      .toSet ++ Set(
+      "agglomerate_view_5",
+      "agglomerate_view_10",
+      "agglomerate_view_15",
+      "agglomerate_view_20",
+      "agglomerate_view_25",
+      "agglomerate_view_30",
+      "agglomerate_view_35",
+      "agglomerate_view_40",
+      "agglomerate_view_45",
+      "agglomerate_view_50",
+      "agglomerate_view_55",
+      "agglomerate_view_60",
+      "agglomerate_view_65",
+      "agglomerate_view_70",
+      "agglomerate_view_75",
+      "agglomerate_view_80",
+      "agglomerate_view_85",
+      "agglomerate_view_90",
+      "agglomerate_view_95",
+      "agglomerate_view_100"
+    )
   }
 
   def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(implicit ec: ExecutionContext,

From 90d97cd2556158c0fb68f3c271fd243a01a65c92 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 28 May 2025 10:41:39 +0200
Subject: [PATCH 17/22] shortcut for single-dimension shape+offset

---
 .../datastore/controllers/Application.scala   |  9 ++---
 .../DatasetArrayBucketProvider.scala          |  4 +--
 .../datastore/datareaders/DatasetArray.scala  | 36 ++++++++++---------
 .../datareaders/MultiArrayUtils.scala         |  3 +-
 .../services/AgglomerateService.scala         | 32 +++++++----------
 5 files changed, 37 insertions(+), 47 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
index 7a3802c5e64..90e222df1be 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
@@ -3,13 +3,8 @@ package com.scalableminds.webknossos.datastore.controllers
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.helpers.NativeBucketScanner
-import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, ElementClass}
-import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
-import com.scalableminds.webknossos.datastore.services.{
-  AgglomerateService,
-  ApplicationHealthService,
-  ZarrAgglomerateService
-}
+import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
+import com.scalableminds.webknossos.datastore.services.{ApplicationHealthService, ZarrAgglomerateService}
 import com.scalableminds.webknossos.datastore.storage.DataStoreRedisStore
 import net.liftweb.common.Box.tryo
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
index 0c78fa32c7c..bd46b455e98 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
@@ -40,8 +40,8 @@ class DatasetArrayBucketProvider(dataLayer: DataLayer,
       bucket = readInstruction.bucket
       shape = Vec3Int.full(bucket.bucketLength)
       offset = Vec3Int(bucket.topLeft.voxelXInMag, bucket.topLeft.voxelYInMag, bucket.topLeft.voxelZInMag)
-      bucketData <- datasetArray.readBytesWithAdditionalCoordinates(shape,
-                                                                    offset,
+      bucketData <- datasetArray.readBytesWithAdditionalCoordinates(offset,
+                                                                    shape,
                                                                     bucket.additionalCoordinates,
                                                                     dataLayer.elementClass == ElementClass.uint24)
     } yield bucketData
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
index dfa848800f4..21c77270b7f 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
@@ -3,7 +3,6 @@ package com.scalableminds.webknossos.datastore.datareaders
 import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.cache.AlfuCache
 import com.scalableminds.util.geometry.Vec3Int
-import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.datavault.VaultPath
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
@@ -69,33 +68,33 @@ class DatasetArray(vaultPath: VaultPath,
     }
 
   def readBytesWithAdditionalCoordinates(
-      shapeXYZ: Vec3Int,
       offsetXYZ: Vec3Int,
+      shapeXYZ: Vec3Int,
       additionalCoordinatesOpt: Option[Seq[AdditionalCoordinate]],
       shouldReadUint24: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Array[Byte]] =
     for {
-      (shapeArray, offsetArray) <- tryo(constructShapeAndOffsetArrays(
-        shapeXYZ,
+      (offsetArray, shapeArray) <- tryo(constructShapeAndOffsetArrays(
         offsetXYZ,
+        shapeXYZ,
         additionalCoordinatesOpt,
         shouldReadUint24)).toFox ?~> "failed to construct shape and offset array for requested coordinates"
-      bytes <- readBytes(shapeArray, offsetArray)
+      bytes <- readBytes(offsetArray, shapeArray)
     } yield bytes
 
-  private def constructShapeAndOffsetArrays(shapeXYZ: Vec3Int,
-                                            offsetXYZ: Vec3Int,
+  private def constructShapeAndOffsetArrays(offsetXYZ: Vec3Int,
+                                            shapeXYZ: Vec3Int,
                                             additionalCoordinatesOpt: Option[Seq[AdditionalCoordinate]],
                                             shouldReadUint24: Boolean): (Array[Int], Array[Int]) = {
-    val shapeArray: Array[Int] = Array.fill(rank)(1)
-    shapeArray(rank - 3) = shapeXYZ.x
-    shapeArray(rank - 2) = shapeXYZ.y
-    shapeArray(rank - 1) = shapeXYZ.z
-
     val offsetArray: Array[Int] = Array.fill(rank)(0)
     offsetArray(rank - 3) = offsetXYZ.x
     offsetArray(rank - 2) = offsetXYZ.y
     offsetArray(rank - 1) = offsetXYZ.z
 
+    val shapeArray: Array[Int] = Array.fill(rank)(1)
+    shapeArray(rank - 3) = shapeXYZ.x
+    shapeArray(rank - 2) = shapeXYZ.y
+    shapeArray(rank - 1) = shapeXYZ.z
+
     axisOrder.c.foreach { channelAxisInner =>
       val channelAxisOuter = fullAxisOrder.arrayToWkPermutation(channelAxisInner)
       // If a channelIndex is requested, and a channel axis is known, add an offset to the channel axis
@@ -115,14 +114,14 @@ class DatasetArray(vaultPath: VaultPath,
         // shapeArray at positions of additional coordinates is always 1
       }
     }
-    (shapeArray, offsetArray)
+    (offsetArray, shapeArray)
   }
 
   // returns byte array in fortran-order with little-endian values
-  private def readBytes(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext,
+  private def readBytes(offset: Array[Int], shape: Array[Int])(implicit ec: ExecutionContext,
                                                                tc: TokenContext): Fox[Array[Byte]] =
     for {
-      typedMultiArray <- readAsFortranOrder(shape, offset)
+      typedMultiArray <- readAsFortranOrder(offset, shape)
       asBytes <- BytesConverter.toByteArray(typedMultiArray, header.resolvedDataType, ByteOrder.LITTLE_ENDIAN).toFox
     } yield asBytes
 
@@ -153,7 +152,7 @@ class DatasetArray(vaultPath: VaultPath,
   // The local variables like chunkIndices are also in this order unless explicitly named.
   // Loading data adapts to the array's axis order so that …CXYZ data in fortran-order is
   // returned, regardless of the array’s internal storage.
-  private def readAsFortranOrder(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext,
+  private def readAsFortranOrder(offset: Array[Int], shape: Array[Int])(implicit ec: ExecutionContext,
                                                                         tc: TokenContext): Fox[MultiArray] = {
     val totalOffset: Array[Int] = offset.zip(header.voxelOffset).map { case (o, v) => o - v }.padTo(offset.length, 0)
     val chunkIndices = ChunkUtils.computeChunkIndices(
@@ -189,7 +188,10 @@ class DatasetArray(vaultPath: VaultPath,
     }
   }
 
-  def readAsMultiArray(shape: Array[Int], offset: Array[Long])(implicit ec: ExecutionContext,
+  def readAsMultiArray(offset: Long, shape: Int)(implicit ec: ExecutionContext, tc: TokenContext): Fox[MultiArray] =
+    readAsMultiArray(Array(offset), Array(shape))
+
+  def readAsMultiArray(offset: Array[Long], shape: Array[Int])(implicit ec: ExecutionContext,
                                                                tc: TokenContext): Fox[MultiArray] =
     if (shape.product == 0) {
       Fox.successful(MultiArrayUtils.createEmpty(rank))
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
index a9a2160b7a6..69a3990e68b 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
@@ -45,9 +45,8 @@ object MultiArrayUtils extends LazyLogging {
     }
   }
 
-  def createEmpty(rank: Int): MultiArray = {
+  def createEmpty(rank: Int): MultiArray =
     MultiArray.factory(MADataType.FLOAT, Array.fill(rank)(0))
-  }
 
   /**
     * Offset describes the displacement between source and target array.<br/>
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 94daad2d85a..44eebd6b09e 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -62,7 +62,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
   private def mapSingleSegment(segmentToAgglomerate: DatasetArray, segmentId: Long)(implicit ec: ExecutionContext,
                                                                                     tc: TokenContext): Fox[Long] =
     for {
-      asMultiArray <- segmentToAgglomerate.readAsMultiArray(shape = Array(1), offset = Array(segmentId))
+      asMultiArray <- segmentToAgglomerate.readAsMultiArray(offset = segmentId, shape = 1)
     } yield asMultiArray.getLong(0)
 
   private def openZarrArrayCached(agglomerateFileKey: AgglomerateFileKey,
@@ -147,10 +147,8 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
       agglomerateToEdgesOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges_offsets")
 
-      positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = Array(agglomerateId),
-                                                                                  shape = Array(2))
-      edgesRange: MultiArray <- agglomerateToEdgesOffsets.readAsMultiArray(offset = Array(agglomerateId),
-                                                                           shape = Array(2))
+      positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
+      edgesRange: MultiArray <- agglomerateToEdgesOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
       nodeCount = positionsRange.getLong(1) - positionsRange.getLong(0)
       edgeCount = edgesRange.getLong(1) - edgesRange.getLong(0)
       edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
@@ -223,10 +221,8 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
       agglomerateToEdgesOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges_offsets")
 
-      positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(shape = Array(2),
-                                                                                  offset = Array(agglomerateId))
-      edgesRange: MultiArray <- agglomerateToEdgesOffsets.readAsMultiArray(shape = Array(2),
-                                                                           offset = Array(agglomerateId))
+      positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
+      edgesRange: MultiArray <- agglomerateToEdgesOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
       nodeCount = positionsRange.getLong(1) - positionsRange.getLong(0)
       edgeCount = edgesRange.getLong(1) - edgesRange.getLong(0)
       edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
@@ -236,14 +232,14 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       positions: MultiArray <- agglomerateToPositions.readAsMultiArray(offset = Array(positionsRange.getLong(0), 0),
                                                                        shape = Array(nodeCount.toInt, 3))
       agglomerateToSegments <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments")
-      segmentIds: MultiArray <- agglomerateToSegments.readAsMultiArray(offset = Array(positionsRange.getInt(0)),
-                                                                       shape = Array(nodeCount.toInt))
+      segmentIds: MultiArray <- agglomerateToSegments.readAsMultiArray(offset = positionsRange.getInt(0),
+                                                                       shape = nodeCount.toInt)
       agglomerateToEdges <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges")
       edges: MultiArray <- agglomerateToEdges.readAsMultiArray(offset = Array(edgesRange.getLong(0), 0),
                                                                shape = Array(edgeCount.toInt, 2))
       agglomerateToAffinities <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_affinities")
-      affinities: MultiArray <- agglomerateToAffinities.readAsMultiArray(offset = Array(edgesRange.getLong(0)),
-                                                                         shape = Array(edgeCount.toInt))
+      affinities: MultiArray <- agglomerateToAffinities.readAsMultiArray(offset = edgesRange.getLong(0),
+                                                                         shape = edgeCount.toInt)
 
       agglomerateGraph = AgglomerateGraph(
         // unsafeWrapArray is fine, because the underlying arrays are never mutated
@@ -270,13 +266,12 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
     for {
       agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
       agglomerateToSegments <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments")
-      segmentRange <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = Array(agglomerateId), shape = Array(2))
+      segmentRange <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
       segmentCount = segmentRange.getLong(1) - segmentRange.getLong(0)
       segmentIds <- if (segmentCount == 0)
         Fox.successful(MultiArray.factory(DataType.LONG, Array(0, 0)))
       else
-        agglomerateToSegments.readAsMultiArray(offset = Array(segmentRange.getLong(0)),
-                                               shape = Array(segmentCount.toInt))
+        agglomerateToSegments.readAsMultiArray(offset = segmentRange.getLong(0), shape = segmentCount.toInt)
     } yield segmentIds.getStorage.asInstanceOf[Array[Long]].toSeq
 
   def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: Seq[Long])(
@@ -295,8 +290,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       segmentToAgglomerate <- openZarrArrayCached(agglomerateFileKey, "segment_to_agglomerate")
       agglomerateId <- mapSingleSegment(segmentToAgglomerate, segmentId)
       agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
-      segmentsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = Array(agglomerateId),
-                                                                                 shape = Array(2))
+      segmentsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
       agglomerateToSegments <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments")
       segmentIndex <- binarySearchForSegment(segmentsRange.getLong(0),
                                              segmentsRange.getLong(1),
@@ -315,7 +309,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
     else {
       val middle = rangeStart + (rangeEnd - rangeStart) / 2
       for {
-        segmentIdAtMiddleMA <- agglomerateToSegments.readAsMultiArray(offset = Array(middle), shape = Array(1))
+        segmentIdAtMiddleMA <- agglomerateToSegments.readAsMultiArray(offset = middle, shape = 1)
         segmentIdAtMiddle = segmentIdAtMiddleMA.getLong(0)
         segmentIndex <- if (segmentIdAtMiddle == segmentId)
           Fox.successful(middle)

From 8551f99f82480d5df908e3cccb56a5901fec5805 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 28 May 2025 11:04:52 +0200
Subject: [PATCH 18/22] handle uint32 agglomerate_to_segments arrays

---
 .../DatasetArrayBucketProvider.scala          |  2 +-
 .../datareaders/MultiArrayUtils.scala         | 16 +++++++++++-
 .../services/AgglomerateService.scala         | 26 +++++++++++--------
 .../services/mesh/AdHocMeshService.scala      |  2 +-
 4 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
index bd46b455e98..1e9d572e4ed 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
@@ -38,8 +38,8 @@ class DatasetArrayBucketProvider(dataLayer: DataLayer,
       datasetArray <- datasetArrayCache.getOrLoad(readInstruction.bucket.mag,
                                                   _ => openDatasetArrayWithTimeLogging(readInstruction))
       bucket = readInstruction.bucket
-      shape = Vec3Int.full(bucket.bucketLength)
       offset = Vec3Int(bucket.topLeft.voxelXInMag, bucket.topLeft.voxelYInMag, bucket.topLeft.voxelZInMag)
+      shape = Vec3Int.full(bucket.bucketLength)
       bucketData <- datasetArray.readBytesWithAdditionalCoordinates(offset,
                                                                     shape,
                                                                     bucket.additionalCoordinates,
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
index 69a3990e68b..86e19397b85 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
@@ -2,7 +2,7 @@ package com.scalableminds.webknossos.datastore.datareaders
 
 import ArrayDataType.ArrayDataType
 import com.typesafe.scalalogging.LazyLogging
-import net.liftweb.common.Box
+import net.liftweb.common.{Box, Failure, Full}
 import net.liftweb.common.Box.tryo
 import ucar.ma2.{IndexIterator, InvalidRangeException, Range, Array => MultiArray, DataType => MADataType}
 
@@ -48,6 +48,20 @@ object MultiArrayUtils extends LazyLogging {
   def createEmpty(rank: Int): MultiArray =
     MultiArray.factory(MADataType.FLOAT, Array.fill(rank)(0))
 
+  def toLongArray(multiArray: MultiArray): Box[Array[Long]] =
+    multiArray.getDataType match {
+      case MADataType.LONG | MADataType.ULONG =>
+        Full(multiArray.getStorage.asInstanceOf[Array[Long]])
+      case MADataType.INT =>
+        Full(multiArray.getStorage.asInstanceOf[Array[Int]].map(_.toLong))
+      case MADataType.UINT =>
+        Full(multiArray.getStorage.asInstanceOf[Array[Int]].map { signed =>
+          if (signed >= 0) signed.toLong else signed.toLong + Int.MaxValue.toLong + Int.MaxValue.toLong + 2L
+        })
+      case _ =>
+        Failure("Cannot convert MultiArray to LongArray: unsupported data type.")
+    }
+
   /**
     * Offset describes the displacement between source and target array.<br/>
     * <br/>
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 44eebd6b09e..a754e42c0cb 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -10,7 +10,7 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.AgglomerateGraph.{AgglomerateEdge, AgglomerateGraph}
 import com.scalableminds.webknossos.datastore.DataStoreConfig
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, SkeletonTracing, Tree, TreeTypeProto}
-import com.scalableminds.webknossos.datastore.datareaders.DatasetArray
+import com.scalableminds.webknossos.datastore.datareaders.{DatasetArray, MultiArrayUtils}
 import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3Array
 import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto
 import com.scalableminds.webknossos.datastore.helpers.{NativeBucketScanner, NodeDefaults, SkeletonTracingDefaults}
@@ -21,7 +21,7 @@ import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.{Box, Failure, Full}
 import net.liftweb.common.Box.tryo
 import org.apache.commons.io.FilenameUtils
-import ucar.ma2.{DataType, Array => MultiArray}
+import ucar.ma2.{Array => MultiArray}
 
 import java.net.URI
 import java.nio._
@@ -232,8 +232,9 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       positions: MultiArray <- agglomerateToPositions.readAsMultiArray(offset = Array(positionsRange.getLong(0), 0),
                                                                        shape = Array(nodeCount.toInt, 3))
       agglomerateToSegments <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments")
-      segmentIds: MultiArray <- agglomerateToSegments.readAsMultiArray(offset = positionsRange.getInt(0),
-                                                                       shape = nodeCount.toInt)
+      segmentIdsMA: MultiArray <- agglomerateToSegments.readAsMultiArray(offset = positionsRange.getInt(0),
+                                                                         shape = nodeCount.toInt)
+      segmentIds: Array[Long] <- MultiArrayUtils.toLongArray(segmentIdsMA).toFox
       agglomerateToEdges <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges")
       edges: MultiArray <- agglomerateToEdges.readAsMultiArray(offset = Array(edgesRange.getLong(0), 0),
                                                                shape = Array(edgeCount.toInt, 2))
@@ -243,11 +244,11 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
 
       agglomerateGraph = AgglomerateGraph(
         // unsafeWrapArray is fine, because the underlying arrays are never mutated
-        segments = ArraySeq.unsafeWrapArray(segmentIds.getStorage.asInstanceOf[Array[Long]]),
+        segments = ArraySeq.unsafeWrapArray(segmentIds),
         edges = (0 until edges.getShape()(0)).map { edgeIdx: Int =>
           AgglomerateEdge(
-            source = segmentIds.getLong(edges.getInt(edges.getIndex.set(Array(edgeIdx, 0)))),
-            target = segmentIds.getLong(edges.getInt(edges.getIndex.set(Array(edgeIdx, 1))))
+            source = segmentIds(edges.getInt(edges.getIndex.set(Array(edgeIdx, 0)))),
+            target = segmentIds(edges.getInt(edges.getIndex.set(Array(edgeIdx, 1))))
           )
         },
         positions = (0 until nodeCount.toInt).map { nodeIdx: Int =>
@@ -269,10 +270,12 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       segmentRange <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
       segmentCount = segmentRange.getLong(1) - segmentRange.getLong(0)
       segmentIds <- if (segmentCount == 0)
-        Fox.successful(MultiArray.factory(DataType.LONG, Array(0, 0)))
+        Fox.successful(Array.empty[Long])
       else
-        agglomerateToSegments.readAsMultiArray(offset = segmentRange.getLong(0), shape = segmentCount.toInt)
-    } yield segmentIds.getStorage.asInstanceOf[Array[Long]].toSeq
+        agglomerateToSegments
+          .readAsMultiArray(offset = segmentRange.getLong(0), shape = segmentCount.toInt)
+          .flatMap(MultiArrayUtils.toLongArray(_).toFox)
+    } yield segmentIds.toSeq
 
   def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: Seq[Long])(
       implicit ec: ExecutionContext,
@@ -310,7 +313,8 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
       val middle = rangeStart + (rangeEnd - rangeStart) / 2
       for {
         segmentIdAtMiddleMA <- agglomerateToSegments.readAsMultiArray(offset = middle, shape = 1)
-        segmentIdAtMiddle = segmentIdAtMiddleMA.getLong(0)
+        segmentIdAdMiddleArray: Array[Long] <- MultiArrayUtils.toLongArray(segmentIdAtMiddleMA).toFox
+        segmentIdAtMiddle = segmentIdAdMiddleArray(0)
         segmentIndex <- if (segmentIdAtMiddle == segmentId)
           Fox.successful(middle)
         else if (segmentIdAtMiddle < segmentId) {
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala
index cf8f5c2a135..6e711449df3 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala
@@ -14,7 +14,7 @@ import com.scalableminds.webknossos.datastore.models.requests.{
 import com.scalableminds.webknossos.datastore.services.mcubes.MarchingCubes
 import com.scalableminds.webknossos.datastore.services.{BinaryDataService, MappingService}
 import com.typesafe.scalalogging.LazyLogging
-import net.liftweb.common.{Box, Failure, Full}
+import net.liftweb.common.{Box, Failure}
 import org.apache.pekko.actor.{Actor, ActorRef, ActorSystem, Props}
 import org.apache.pekko.pattern.ask
 import org.apache.pekko.routing.RoundRobinPool

From d183c99d2e486b834581f99723cc424dbb7e08f7 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 28 May 2025 11:14:56 +0200
Subject: [PATCH 19/22] useZarr=false to test ci

---
 .../webknossos/datastore/services/AgglomerateService.scala     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index a754e42c0cb..32d7dc954d2 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -339,8 +339,9 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
   private val cumsumFileName = "cumsum.json"
 
   // TODO remove
-  private val useZarr = true
+  private val useZarr = false
 
+  // TODO clear on reload
   lazy val agglomerateFileCache = new AgglomerateFileCache(config.Datastore.Cache.AgglomerateFile.maxFileHandleEntries)
 
   def exploreAgglomerates(organizationId: String, datasetDirectoryName: String, dataLayerName: String): Set[String] = {

From 8335196c6ac143d8943fc7a46b1cdb5c58664195 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 28 May 2025 13:19:15 +0200
Subject: [PATCH 20/22] undo some changes, trying to create minimal example of
 id bug

---
 .../DatasetArrayBucketProvider.scala          |   4 +-
 .../datastore/dataformats/wkw/WKWHeader.scala |   2 +-
 .../datastore/datareaders/ChunkUtils.scala    |   4 +-
 .../datastore/datareaders/DatasetArray.scala  | 113 ++-------
 .../datastore/datareaders/DatasetHeader.scala |   2 +-
 .../datastore/datareaders/n5/N5Header.scala   |   4 +-
 .../precomputed/PrecomputedHeader.scala       |   4 +-
 .../datastore/datareaders/wkw/WKWArray.scala  |   2 +-
 .../datareaders/zarr/ZarrHeader.scala         |   6 +-
 .../datareaders/zarr3/Zarr3ArrayHeader.scala  |   8 +-
 .../explore/NgffExplorationUtils.scala        |   2 +-
 .../datastore/explore/NgffV0_4Explorer.scala  |   4 +-
 .../datastore/explore/NgffV0_5Explorer.scala  |   2 +-
 .../services/AgglomerateService.scala         | 225 +-----------------
 ...VolumeTracingZarrStreamingController.scala |   6 +-
 15 files changed, 61 insertions(+), 327 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
index 1e9d572e4ed..86f51996b3d 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
@@ -40,8 +40,8 @@ class DatasetArrayBucketProvider(dataLayer: DataLayer,
       bucket = readInstruction.bucket
       offset = Vec3Int(bucket.topLeft.voxelXInMag, bucket.topLeft.voxelYInMag, bucket.topLeft.voxelZInMag)
       shape = Vec3Int.full(bucket.bucketLength)
-      bucketData <- datasetArray.readBytesWithAdditionalCoordinates(offset,
-                                                                    shape,
+      bucketData <- datasetArray.readBytesWithAdditionalCoordinates(shape,
+                                                                    offset,
                                                                     bucket.additionalCoordinates,
                                                                     dataLayer.elementClass == ElementClass.uint24)
     } yield bucketData
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWHeader.scala
index adde6bb2817..d694ef96163 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWHeader.scala
@@ -78,7 +78,7 @@ case class WKWHeader(
     }
   }
 
-  override def datasetShape: Option[Array[Long]] = None
+  override def datasetShape: Option[Array[Int]] = None
 
   override def chunkShape: Array[Int] =
     Array(numChannels, numVoxelsPerChunkDimension, numVoxelsPerChunkDimension, numVoxelsPerChunkDimension)
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala
index 8959b15e2bc..b64cd380635 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala
@@ -1,10 +1,10 @@
 package com.scalableminds.webknossos.datastore.datareaders
 
 object ChunkUtils {
-  def computeChunkIndices(arrayShapeOpt: Option[Array[Long]],
+  def computeChunkIndices(arrayShapeOpt: Option[Array[Int]],
                           arrayChunkShape: Array[Int],
                           selectedShape: Array[Int],
-                          selectedOffset: Array[Long]): Seq[Array[Int]] = {
+                          selectedOffset: Array[Int]): Seq[Array[Int]] = {
     val nDims = arrayChunkShape.length
     val start = new Array[Int](nDims)
     val end = new Array[Int](nDims)
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
index 21c77270b7f..0dcbd50a4fc 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
@@ -8,7 +8,6 @@ import com.scalableminds.webknossos.datastore.datavault.VaultPath
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate
 import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis
-import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.Box.tryo
 import ucar.ma2.{Array => MultiArray}
 
@@ -27,8 +26,7 @@ class DatasetArray(vaultPath: VaultPath,
                    channelIndex: Option[Int],
                    additionalAxes: Option[Seq[AdditionalAxis]],
                    sharedChunkContentsCache: AlfuCache[String, MultiArray])
-    extends FoxImplicits
-    with LazyLogging {
+    extends FoxImplicits {
 
   protected lazy val fullAxisOrder: FullAxisOrder =
     FullAxisOrder.fromAxisOrderAndAdditionalAxes(rank, axisOrder, additionalAxes)
@@ -50,7 +48,7 @@ class DatasetArray(vaultPath: VaultPath,
     header.rank + 1
   }
 
-  lazy val datasetShape: Option[Array[Long]] = if (axisOrder.hasZAxis) {
+  lazy val datasetShape: Option[Array[Int]] = if (axisOrder.hasZAxis) {
     header.datasetShape
   } else {
     header.datasetShape.map(shape => shape :+ 1)
@@ -68,33 +66,33 @@ class DatasetArray(vaultPath: VaultPath,
     }
 
   def readBytesWithAdditionalCoordinates(
-      offsetXYZ: Vec3Int,
       shapeXYZ: Vec3Int,
+      offsetXYZ: Vec3Int,
       additionalCoordinatesOpt: Option[Seq[AdditionalCoordinate]],
       shouldReadUint24: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Array[Byte]] =
     for {
-      (offsetArray, shapeArray) <- tryo(constructShapeAndOffsetArrays(
-        offsetXYZ,
+      (shapeArray, offsetArray) <- tryo(constructShapeAndOffsetArrays(
         shapeXYZ,
+        offsetXYZ,
         additionalCoordinatesOpt,
         shouldReadUint24)).toFox ?~> "failed to construct shape and offset array for requested coordinates"
-      bytes <- readBytes(offsetArray, shapeArray)
+      bytes <- readBytes(shapeArray, offsetArray)
     } yield bytes
 
-  private def constructShapeAndOffsetArrays(offsetXYZ: Vec3Int,
-                                            shapeXYZ: Vec3Int,
+  private def constructShapeAndOffsetArrays(shapeXYZ: Vec3Int,
+                                            offsetXYZ: Vec3Int,
                                             additionalCoordinatesOpt: Option[Seq[AdditionalCoordinate]],
                                             shouldReadUint24: Boolean): (Array[Int], Array[Int]) = {
-    val offsetArray: Array[Int] = Array.fill(rank)(0)
-    offsetArray(rank - 3) = offsetXYZ.x
-    offsetArray(rank - 2) = offsetXYZ.y
-    offsetArray(rank - 1) = offsetXYZ.z
-
     val shapeArray: Array[Int] = Array.fill(rank)(1)
     shapeArray(rank - 3) = shapeXYZ.x
     shapeArray(rank - 2) = shapeXYZ.y
     shapeArray(rank - 1) = shapeXYZ.z
 
+    val offsetArray: Array[Int] = Array.fill(rank)(0)
+    offsetArray(rank - 3) = offsetXYZ.x
+    offsetArray(rank - 2) = offsetXYZ.y
+    offsetArray(rank - 1) = offsetXYZ.z
+
     axisOrder.c.foreach { channelAxisInner =>
       val channelAxisOuter = fullAxisOrder.arrayToWkPermutation(channelAxisInner)
       // If a channelIndex is requested, and a channel axis is known, add an offset to the channel axis
@@ -114,14 +112,14 @@ class DatasetArray(vaultPath: VaultPath,
         // shapeArray at positions of additional coordinates is always 1
       }
     }
-    (offsetArray, shapeArray)
+    (shapeArray, offsetArray)
   }
 
   // returns byte array in fortran-order with little-endian values
-  private def readBytes(offset: Array[Int], shape: Array[Int])(implicit ec: ExecutionContext,
+  private def readBytes(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext,
                                                                tc: TokenContext): Fox[Array[Byte]] =
     for {
-      typedMultiArray <- readAsFortranOrder(offset, shape)
+      typedMultiArray <- readAsFortranOrder(shape, offset)
       asBytes <- BytesConverter.toByteArray(typedMultiArray, header.resolvedDataType, ByteOrder.LITTLE_ENDIAN).toFox
     } yield asBytes
 
@@ -152,16 +150,14 @@ class DatasetArray(vaultPath: VaultPath,
   // The local variables like chunkIndices are also in this order unless explicitly named.
   // Loading data adapts to the array's axis order so that …CXYZ data in fortran-order is
   // returned, regardless of the array’s internal storage.
-  private def readAsFortranOrder(offset: Array[Int], shape: Array[Int])(implicit ec: ExecutionContext,
+  private def readAsFortranOrder(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext,
                                                                         tc: TokenContext): Fox[MultiArray] = {
     val totalOffset: Array[Int] = offset.zip(header.voxelOffset).map { case (o, v) => o - v }.padTo(offset.length, 0)
-    val chunkIndices = ChunkUtils.computeChunkIndices(
-      datasetShape.map(fullAxisOrder.permuteIndicesArrayToWkLong),
-      fullAxisOrder.permuteIndicesArrayToWk(chunkShape),
-      shape,
-      totalOffset.map(_.toLong)
-    )
-    if (partialCopyingIsNotNeededForWkOrder(shape, totalOffset, chunkIndices)) {
+    val chunkIndices = ChunkUtils.computeChunkIndices(datasetShape.map(fullAxisOrder.permuteIndicesArrayToWk),
+                                                      fullAxisOrder.permuteIndicesArrayToWk(chunkShape),
+                                                      shape,
+                                                      totalOffset)
+    if (partialCopyingIsNotNeeded(shape, totalOffset, chunkIndices)) {
       for {
         chunkIndex <- chunkIndices.headOption.toFox
         sourceChunk: MultiArray <- getSourceChunkDataWithCache(fullAxisOrder.permuteIndicesWkToArray(chunkIndex),
@@ -188,50 +184,10 @@ class DatasetArray(vaultPath: VaultPath,
     }
   }
 
-  def readAsMultiArray(offset: Long, shape: Int)(implicit ec: ExecutionContext, tc: TokenContext): Fox[MultiArray] =
-    readAsMultiArray(Array(offset), Array(shape))
-
-  def readAsMultiArray(offset: Array[Long], shape: Array[Int])(implicit ec: ExecutionContext,
-                                                               tc: TokenContext): Fox[MultiArray] =
-    if (shape.product == 0) {
-      Fox.successful(MultiArrayUtils.createEmpty(rank))
-    } else {
-      val totalOffset: Array[Long] = offset.zip(header.voxelOffset).map { case (o, v) => o - v }.padTo(offset.length, 0)
-      val chunkIndices = ChunkUtils.computeChunkIndices(datasetShape, chunkShape, shape, totalOffset)
-      if (partialCopyingIsNotNeededForMultiArray(shape, totalOffset, chunkIndices)) {
-        for {
-          chunkIndex <- chunkIndices.headOption.toFox
-          sourceChunk: MultiArray <- getSourceChunkDataWithCache(chunkIndex, useSkipTypingShortcut = true)
-        } yield sourceChunk
-      } else {
-        val targetBuffer = MultiArrayUtils.createDataBuffer(header.resolvedDataType, shape)
-        val targetMultiArray = MultiArrayUtils.createArrayWithGivenStorage(targetBuffer, shape)
-        val copiedFuture = Fox.combined(chunkIndices.map { chunkIndex: Array[Int] =>
-          for {
-            sourceChunk: MultiArray <- getSourceChunkDataWithCache(chunkIndex)
-            offsetInChunk = computeOffsetInChunkIgnoringAxisOrder(chunkIndex, totalOffset)
-            _ <- tryo(MultiArrayUtils.copyRange(offsetInChunk, sourceChunk, targetMultiArray)).toFox ?~> formatCopyRangeErrorWithoutAxisOrder(
-              offsetInChunk,
-              sourceChunk,
-              targetMultiArray)
-          } yield ()
-        })
-        for {
-          _ <- copiedFuture
-        } yield targetMultiArray
-      }
-    }
-
   private def formatCopyRangeError(offsetInChunk: Array[Int], sourceChunk: MultiArray, target: MultiArray): String =
     s"Copying data from dataset chunk failed. Chunk shape (F): ${printAsOuterF(sourceChunk.getShape)}, target shape (F): ${printAsOuterF(
       target.getShape)}, offsetInChunk: ${printAsOuterF(offsetInChunk)}. Axis order (C): $fullAxisOrder (outer: ${fullAxisOrder.toStringWk})"
 
-  private def formatCopyRangeErrorWithoutAxisOrder(offsetInChunk: Array[Int],
-                                                   sourceChunk: MultiArray,
-                                                   target: MultiArray): String =
-    s"Copying data from dataset chunk failed. Chunk shape ${sourceChunk.getShape.mkString(",")}, target shape ${target.getShape
-      .mkString(",")}, offsetInChunk: ${offsetInChunk.mkString(",")}"
-
   protected def getShardedChunkPathAndRange(
       chunkIndex: Array[Int])(implicit ec: ExecutionContext, tc: TokenContext): Fox[(VaultPath, NumericRange[Long])] =
     ??? // Defined in subclass
@@ -268,20 +224,9 @@ class DatasetArray(vaultPath: VaultPath,
       chunkIndex.drop(1).mkString(header.dimension_separator.toString) // (c),x,y,z -> z is dropped in 2d case
     }
 
-  private def partialCopyingIsNotNeededForMultiArray(bufferShape: Array[Int],
-                                                     globalOffset: Array[Long],
-                                                     chunkIndices: Seq[Array[Int]]): Boolean =
-    chunkIndices match {
-      case chunkIndex :: Nil =>
-        val offsetInChunk = computeOffsetInChunkIgnoringAxisOrder(chunkIndex, globalOffset)
-        isZeroOffset(offsetInChunk) &&
-        isBufferShapeEqualChunkShape(bufferShape)
-      case _ => false
-    }
-
-  private def partialCopyingIsNotNeededForWkOrder(bufferShape: Array[Int],
-                                                  globalOffset: Array[Int],
-                                                  chunkIndices: Seq[Array[Int]]): Boolean =
+  private def partialCopyingIsNotNeeded(bufferShape: Array[Int],
+                                        globalOffset: Array[Int],
+                                        chunkIndices: Seq[Array[Int]]): Boolean =
     chunkIndices match {
       case chunkIndex :: Nil =>
         val offsetInChunk = computeOffsetInChunk(chunkIndex, globalOffset)
@@ -303,14 +248,8 @@ class DatasetArray(vaultPath: VaultPath,
       globalOffset(dim) - (chunkIndex(dim) * fullAxisOrder.permuteIndicesArrayToWk(chunkShape)(dim))
     }.toArray
 
-  private def computeOffsetInChunkIgnoringAxisOrder(chunkIndex: Array[Int], globalOffset: Array[Long]): Array[Int] =
-    chunkIndex.indices.map { dim =>
-      (globalOffset(dim) - (chunkIndex(dim).toLong * chunkShape(dim).toLong)).toInt
-    }.toArray
-
-  // TODO works only for wk dataet arrays, not agglomerate files
   override def toString: String =
-    s"${getClass.getCanonicalName} fullAxisOrder=$fullAxisOrder shape=${header.datasetShape.map(s => printAsInner(s.map(_.toInt)))} chunkShape=${printAsInner(
+    s"${getClass.getCanonicalName} fullAxisOrder=$fullAxisOrder shape=${header.datasetShape.map(s => printAsInner(s))} chunkShape=${printAsInner(
       header.chunkShape)} dtype=${header.resolvedDataType} fillValue=${header.fillValueNumber}, ${header.compressorImpl}, byteOrder=${header.byteOrder}, vault=${vaultPath.summary}}"
 
 }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
index 3935b42ea6b..0756011f2f9 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala
@@ -11,7 +11,7 @@ import java.nio.ByteOrder
 trait DatasetHeader {
 
   // Note that in DatasetArray, datasetShape and chunkShape are adapted for 2d datasets
-  def datasetShape: Option[Array[Long]] // shape of the entire array
+  def datasetShape: Option[Array[Int]] // shape of the entire array
   def chunkShape: Array[Int] // shape of each chunk,
 
   def dimension_separator: DimensionSeparator
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5Header.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5Header.scala
index 8e178636971..7cc5542b940 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5Header.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/N5Header.scala
@@ -16,7 +16,7 @@ object N5BlockHeader {
 }
 
 case class N5Header(
-    dimensions: Array[Long], // shape of the entire array
+    dimensions: Array[Int], // shape of the entire array
     blockSize: Array[Int], // shape of each chunk
     compression: Option[Map[String, CompressionSetting]] = None, // specifies compressor to use, with parameters
     dataType: String,
@@ -25,7 +25,7 @@ case class N5Header(
   val fill_value: Either[String, Number] = Right(0)
   val order: ArrayOrder = ArrayOrder.F
 
-  override lazy val datasetShape: Option[Array[Long]] = Some(dimensions)
+  override lazy val datasetShape: Option[Array[Int]] = Some(dimensions)
 
   lazy val chunkShape: Array[Int] = blockSize
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala
index ac05bd26556..79073ae62d5 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedHeader.scala
@@ -28,7 +28,7 @@ case class PrecomputedHeader(`type`: String,
 }
 
 case class PrecomputedScale(key: String,
-                            size: Array[Long],
+                            size: Array[Int],
                             resolution: Array[Double],
                             chunk_sizes: Array[Array[Int]],
                             encoding: String,
@@ -45,7 +45,7 @@ case class PrecomputedScale(key: String,
 
 case class PrecomputedScaleHeader(precomputedScale: PrecomputedScale, precomputedHeader: PrecomputedHeader)
     extends DatasetHeader {
-  override def datasetShape: Option[Array[Long]] = Some(precomputedScale.size)
+  override def datasetShape: Option[Array[Int]] = Some(precomputedScale.size)
 
   override def chunkShape: Array[Int] = precomputedScale.chunk_sizes.head
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
index 31242525413..f5e7232f9f1 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/wkw/WKWArray.scala
@@ -129,7 +129,7 @@ class WKWArray(vaultPath: VaultPath,
 
   private def chunkIndexToShardIndex(chunkIndex: Array[Int]) =
     ChunkUtils.computeChunkIndices(
-      header.datasetShape.map(fullAxisOrder.permuteIndicesArrayToWkLong),
+      header.datasetShape.map(fullAxisOrder.permuteIndicesArrayToWk),
       fullAxisOrder.permuteIndicesArrayToWk(header.shardShape),
       header.chunkShape,
       chunkIndex.zip(header.chunkShape).map { case (i, s) => i * s }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/ZarrHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/ZarrHeader.scala
index 6aaf01e431e..66fe7090deb 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/ZarrHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/ZarrHeader.scala
@@ -21,7 +21,7 @@ import play.api.libs.json._
 
 case class ZarrHeader(
     zarr_format: Int, // format version number
-    shape: Array[Long], // shape of the entire array
+    shape: Array[Int], // shape of the entire array
     chunks: Array[Int], // shape of each chunk
     compressor: Option[Map[String, CompressionSetting]] = None, // specifies compressor to use, with parameters
     filters: Option[List[Map[String, String]]] = None, // specifies filters to use, with parameters
@@ -31,7 +31,7 @@ case class ZarrHeader(
     override val order: ArrayOrder
 ) extends DatasetHeader {
 
-  override lazy val datasetShape: Option[Array[Long]] = Some(shape)
+  override lazy val datasetShape: Option[Array[Int]] = Some(shape)
   override lazy val chunkShape: Array[Int] = chunks
 
   override lazy val byteOrder: ByteOrder =
@@ -77,7 +77,7 @@ object ZarrHeader extends JsonImplicits {
     val chunks = Array(channels) ++ additionalAxesChunksEntries ++ Array(cubeLength, cubeLength, cubeLength)
 
     ZarrHeader(zarr_format = 2,
-               shape = shape.map(_.toLong),
+               shape = shape,
                chunks = chunks,
                compressor = compressor,
                dtype = dtype,
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala
index adee8ddcd2c..cbad13779e1 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala
@@ -25,7 +25,7 @@ import java.nio.ByteOrder
 case class Zarr3ArrayHeader(
     zarr_format: Int, // must be 3
     node_type: String, // must be "array"
-    shape: Array[Long],
+    shape: Array[Int],
     data_type: Either[String, ExtensionDataType],
     chunk_grid: Either[ChunkGridSpecification, ExtensionChunkGridSpecification],
     chunk_key_encoding: ChunkKeyEncoding,
@@ -36,7 +36,7 @@ case class Zarr3ArrayHeader(
     dimension_names: Option[Array[String]]
 ) extends DatasetHeader {
 
-  override def datasetShape: Option[Array[Long]] = Some(shape)
+  override def datasetShape: Option[Array[Int]] = Some(shape)
 
   override def chunkShape: Array[Int] = getChunkSize
 
@@ -168,7 +168,7 @@ object Zarr3ArrayHeader extends JsonImplicits {
       for {
         zarr_format <- (json \ "zarr_format").validate[Int]
         node_type <- (json \ "node_type").validate[String]
-        shape <- (json \ "shape").validate[Array[Long]]
+        shape <- (json \ "shape").validate[Array[Int]]
         data_type <- (json \ "data_type").validate[String]
         chunk_grid <- (json \ "chunk_grid").validate[ChunkGridSpecification]
         chunk_key_encoding <- (json \ "chunk_key_encoding").validate[ChunkKeyEncoding]
@@ -271,7 +271,7 @@ object Zarr3ArrayHeader extends JsonImplicits {
       zarr_format = 3,
       node_type = "array",
       // channel, additional axes, XYZ
-      shape = (Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ xyzBBounds).map(_.toLong),
+      shape = (Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ xyzBBounds),
       data_type = Left(dataLayer.elementClass.toString),
       chunk_grid = Left(
         ChunkGridSpecification(
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
index 32a67a0d798..3bdd02a8861 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorationUtils.scala
@@ -182,7 +182,7 @@ trait NgffExplorationUtils extends FoxImplicits {
     Vec3Double(xFactors.product, yFactors.product, zFactors.product)
   }
 
-  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Long]]
+  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Int]]
 
   protected def createAdditionalAxis(name: String, index: Int, bounds: Array[Int]): Box[AdditionalAxis] =
     for {
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_4Explorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_4Explorer.scala
index 9b40b90427f..7f6991e1ae3 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_4Explorer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_4Explorer.scala
@@ -96,7 +96,7 @@ class NgffV0_4Explorer(implicit val ec: ExecutionContext)
       parsedHeader <- zarrayPath.parseAsJson[ZarrHeader] ?~> s"failed to read zarr header at $zarrayPath"
       header = parsedHeader.shape.length match {
         case 2 =>
-          parsedHeader.copy(shape = parsedHeader.shape ++ Array(1L), chunks = parsedHeader.chunks ++ Array(1))
+          parsedHeader.copy(shape = parsedHeader.shape ++ Array(1), chunks = parsedHeader.chunks ++ Array(1))
         case _ => parsedHeader
       }
     } yield header
@@ -125,7 +125,7 @@ class NgffV0_4Explorer(implicit val ec: ExecutionContext)
         elementClass,
         boundingBox)
 
-  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Long]] =
+  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Int]] =
     for {
       zarrHeader <- getZarrHeader(dataset, path)
       shape = zarrHeader.shape
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_5Explorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_5Explorer.scala
index 6ec2421e76a..3b67e6902bb 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_5Explorer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffV0_5Explorer.scala
@@ -122,7 +122,7 @@ class NgffV0_5Explorer(implicit val ec: ExecutionContext)
         elementClass,
         boundingBox)
 
-  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Long]] =
+  protected def getShape(dataset: NgffDataset, path: VaultPath)(implicit tc: TokenContext): Fox[Array[Int]] =
     for {
       zarrHeader <- getZarrHeader(dataset, path)
       shape = zarrHeader.shape
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 32d7dc954d2..19b39dd10d8 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -60,10 +60,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
   protected lazy val bucketScanner = new NativeBucketScanner()
 
   private def mapSingleSegment(segmentToAgglomerate: DatasetArray, segmentId: Long)(implicit ec: ExecutionContext,
-                                                                                    tc: TokenContext): Fox[Long] =
-    for {
-      asMultiArray <- segmentToAgglomerate.readAsMultiArray(offset = segmentId, shape = 1)
-    } yield asMultiArray.getLong(0)
+                                                                                    tc: TokenContext): Fox[Long] = ???
 
   private def openZarrArrayCached(agglomerateFileKey: AgglomerateFileKey,
                                   zarrArrayName: String)(implicit ec: ExecutionContext, tc: TokenContext) =
@@ -89,239 +86,37 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService
   }
 
   def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(implicit ec: ExecutionContext,
-                                                                           tc: TokenContext): Fox[Array[Byte]] = {
-
-    val agglomerateFileKey = AgglomerateFileKey.fromDataRequest(request)
-
-    def convertToAgglomerate(segmentIds: Array[Long],
-                             relevantAgglomerateMap: Map[Long, Long],
-                             bytesPerElement: Int,
-                             putToBufferFunction: (ByteBuffer, Long) => ByteBuffer): Array[Byte] = {
-      val agglomerateIds = segmentIds.map(relevantAgglomerateMap)
-      agglomerateIds
-        .foldLeft(ByteBuffer.allocate(bytesPerElement * segmentIds.length).order(ByteOrder.LITTLE_ENDIAN))(
-          putToBufferFunction)
-        .array
-    }
-
-    val bytesPerElement = ElementClass.bytesPerElement(request.dataLayer.elementClass)
-    val distinctSegmentIds =
-      bucketScanner.collectSegmentIds(data, bytesPerElement, isSigned = false, skipZeroes = false)
-
-    for {
-      segmentToAgglomerate <- openZarrArrayCached(agglomerateFileKey, "segment_to_agglomerate")
-      beforeBuildMap = Instant.now
-      relevantAgglomerateMap: Map[Long, Long] <- Fox
-        .serialCombined(distinctSegmentIds) { segmentId =>
-          mapSingleSegment(segmentToAgglomerate, segmentId).map((segmentId, _))
-        }
-        .map(_.toMap)
-      _ = Instant.logSince(beforeBuildMap, "build map")
-      mappedBytes: Array[Byte] = convertData(data, request.dataLayer.elementClass) match {
-        case data: Array[Byte] =>
-          val longBuffer = LongBuffer.allocate(data.length)
-          data.foreach(e => longBuffer.put(uByteToLong(e)))
-          convertToAgglomerate(longBuffer.array, relevantAgglomerateMap, bytesPerElement, putByte)
-        case data: Array[Short] =>
-          val longBuffer = LongBuffer.allocate(data.length)
-          data.foreach(e => longBuffer.put(uShortToLong(e)))
-          convertToAgglomerate(longBuffer.array, relevantAgglomerateMap, bytesPerElement, putShort)
-        case data: Array[Int] =>
-          val longBuffer = LongBuffer.allocate(data.length)
-          data.foreach(e => longBuffer.put(uIntToLong(e)))
-          convertToAgglomerate(longBuffer.array, relevantAgglomerateMap, bytesPerElement, putInt)
-        case data: Array[Long] => convertToAgglomerate(data, relevantAgglomerateMap, bytesPerElement, putLong)
-        case _                 => data
-      }
-    } yield mappedBytes
-  }
+                                                                           tc: TokenContext): Fox[Array[Byte]] = ???
 
   def generateSkeleton(organizationId: String,
                        datasetDirectoryName: String,
                        dataLayerName: String,
                        mappingName: String,
-                       agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[SkeletonTracing] =
-    for {
-      before <- Instant.nowFox
-      agglomerateFileKey = AgglomerateFileKey(organizationId, datasetDirectoryName, dataLayerName, mappingName)
-      agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
-      agglomerateToEdgesOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges_offsets")
-
-      positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
-      edgesRange: MultiArray <- agglomerateToEdgesOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
-      nodeCount = positionsRange.getLong(1) - positionsRange.getLong(0)
-      edgeCount = edgesRange.getLong(1) - edgesRange.getLong(0)
-      edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
-      _ <- Fox.fromBool(nodeCount <= edgeLimit) ?~> s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)"
-      _ <- Fox.fromBool(edgeCount <= edgeLimit) ?~> s"Agglomerate has too many edges ($edgeCount > $edgeLimit)"
-      agglomerateToPositions <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_positions")
-      positions <- agglomerateToPositions.readAsMultiArray(offset = Array(positionsRange.getLong(0), 0),
-                                                           shape = Array(nodeCount.toInt, 3))
-      agglomerateToEdges <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges")
-      edges: MultiArray <- agglomerateToEdges.readAsMultiArray(offset = Array(edgesRange.getLong(0), 0),
-                                                               shape = Array(edgeCount.toInt, 2))
-      nodeIdStartAtOneOffset = 1
-
-      // TODO use multiarray index iterators?
-      nodes = (0 until nodeCount.toInt).map { nodeIdx =>
-        NodeDefaults.createInstance.copy(
-          id = nodeIdx + nodeIdStartAtOneOffset,
-          position = Vec3IntProto(
-            positions.getInt(positions.getIndex.set(Array(nodeIdx, 0))),
-            positions.getInt(positions.getIndex.set(Array(nodeIdx, 1))),
-            positions.getInt(positions.getIndex.set(Array(nodeIdx, 2)))
-          )
-        )
-      }
-
-      skeletonEdges = (0 until edges.getShape()(0)).map { edgeIdx =>
-        Edge(
-          source = edges.getInt(edges.getIndex.set(Array(edgeIdx, 0))) + nodeIdStartAtOneOffset,
-          target = edges.getInt(edges.getIndex.set(Array(edgeIdx, 1))) + nodeIdStartAtOneOffset
-        )
-      }
-
-      trees = Seq(
-        Tree(
-          treeId = math.abs(agglomerateId.toInt), // used only to deterministically select tree color
-          createdTimestamp = System.currentTimeMillis(),
-          // unsafeWrapArray is fine, because the underlying arrays are never mutated
-          nodes = nodes,
-          edges = skeletonEdges,
-          name = s"agglomerate $agglomerateId ($mappingName)",
-          `type` = Some(TreeTypeProto.AGGLOMERATE)
-        ))
-
-      skeleton = SkeletonTracingDefaults.createInstance.copy(
-        datasetName = datasetDirectoryName,
-        trees = trees
-      )
-
-      _ = if (Instant.since(before) > (100 milliseconds)) {
-        Instant.logSince(
-          before,
-          s"Generating skeleton from agglomerate file with ${skeletonEdges.length} edges, ${nodes.length} nodes",
-          logger)
-      }
-
-    } yield skeleton
+                       agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[SkeletonTracing] = ???
 
   def largestAgglomerateId(agglomerateFileKey: AgglomerateFileKey)(implicit ec: ExecutionContext,
-                                                                   tc: TokenContext): Fox[Long] =
-    for {
-      array <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
-      shape <- array.datasetShape.toFox ?~> "Could not determine array shape"
-      shapeFirstElement <- tryo(shape(0)).toFox
-    } yield shapeFirstElement
+                                                                   tc: TokenContext): Fox[Long] = ???
 
   def generateAgglomerateGraph(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long)(
       implicit ec: ExecutionContext,
-      tc: TokenContext): Fox[AgglomerateGraph] =
-    for {
-      agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
-      agglomerateToEdgesOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges_offsets")
-
-      positionsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
-      edgesRange: MultiArray <- agglomerateToEdgesOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
-      nodeCount = positionsRange.getLong(1) - positionsRange.getLong(0)
-      edgeCount = edgesRange.getLong(1) - edgesRange.getLong(0)
-      edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
-      _ <- Fox.fromBool(nodeCount <= edgeLimit) ?~> s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)"
-      _ <- Fox.fromBool(edgeCount <= edgeLimit) ?~> s"Agglomerate has too many edges ($edgeCount > $edgeLimit)"
-      agglomerateToPositions <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_positions")
-      positions: MultiArray <- agglomerateToPositions.readAsMultiArray(offset = Array(positionsRange.getLong(0), 0),
-                                                                       shape = Array(nodeCount.toInt, 3))
-      agglomerateToSegments <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments")
-      segmentIdsMA: MultiArray <- agglomerateToSegments.readAsMultiArray(offset = positionsRange.getInt(0),
-                                                                         shape = nodeCount.toInt)
-      segmentIds: Array[Long] <- MultiArrayUtils.toLongArray(segmentIdsMA).toFox
-      agglomerateToEdges <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_edges")
-      edges: MultiArray <- agglomerateToEdges.readAsMultiArray(offset = Array(edgesRange.getLong(0), 0),
-                                                               shape = Array(edgeCount.toInt, 2))
-      agglomerateToAffinities <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_affinities")
-      affinities: MultiArray <- agglomerateToAffinities.readAsMultiArray(offset = edgesRange.getLong(0),
-                                                                         shape = edgeCount.toInt)
-
-      agglomerateGraph = AgglomerateGraph(
-        // unsafeWrapArray is fine, because the underlying arrays are never mutated
-        segments = ArraySeq.unsafeWrapArray(segmentIds),
-        edges = (0 until edges.getShape()(0)).map { edgeIdx: Int =>
-          AgglomerateEdge(
-            source = segmentIds(edges.getInt(edges.getIndex.set(Array(edgeIdx, 0)))),
-            target = segmentIds(edges.getInt(edges.getIndex.set(Array(edgeIdx, 1))))
-          )
-        },
-        positions = (0 until nodeCount.toInt).map { nodeIdx: Int =>
-          Vec3IntProto(
-            positions.getInt(positions.getIndex.set(Array(nodeIdx, 0))),
-            positions.getInt(positions.getIndex.set(Array(nodeIdx, 1))),
-            positions.getInt(positions.getIndex.set(Array(nodeIdx, 2)))
-          )
-        },
-        affinities = ArraySeq.unsafeWrapArray(affinities.getStorage.asInstanceOf[Array[Float]])
-      )
-    } yield agglomerateGraph
+      tc: TokenContext): Fox[AgglomerateGraph] = ???
 
   def segmentIdsForAgglomerateId(agglomerateFileKey: AgglomerateFileKey,
                                  agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Seq[Long]] =
-    for {
-      agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
-      agglomerateToSegments <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments")
-      segmentRange <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
-      segmentCount = segmentRange.getLong(1) - segmentRange.getLong(0)
-      segmentIds <- if (segmentCount == 0)
-        Fox.successful(Array.empty[Long])
-      else
-        agglomerateToSegments
-          .readAsMultiArray(offset = segmentRange.getLong(0), shape = segmentCount.toInt)
-          .flatMap(MultiArrayUtils.toLongArray(_).toFox)
-    } yield segmentIds.toSeq
+    ???
 
   def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: Seq[Long])(
       implicit ec: ExecutionContext,
-      tc: TokenContext): Fox[Seq[Long]] =
-    for {
-      segmentToAgglomerate <- openZarrArrayCached(agglomerateFileKey, "segment_to_agglomerate")
-      agglomerateIds <- Fox.serialCombined(segmentIds) { segmentId =>
-        mapSingleSegment(segmentToAgglomerate, segmentId)
-      }
-    } yield agglomerateIds
+      tc: TokenContext): Fox[Seq[Long]] = ???
 
-  def positionForSegmentId(agglomerateFileKey: AgglomerateFileKey, segmentId: Long)(implicit ec: ExecutionContext,
-                                                                                    tc: TokenContext): Fox[Vec3Int] =
-    for {
-      segmentToAgglomerate <- openZarrArrayCached(agglomerateFileKey, "segment_to_agglomerate")
-      agglomerateId <- mapSingleSegment(segmentToAgglomerate, segmentId)
-      agglomerateToSegmentsOffsets <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments_offsets")
-      segmentsRange: MultiArray <- agglomerateToSegmentsOffsets.readAsMultiArray(offset = agglomerateId, shape = 2)
-      agglomerateToSegments <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_segments")
-      segmentIndex <- binarySearchForSegment(segmentsRange.getLong(0),
-                                             segmentsRange.getLong(1),
-                                             segmentId,
-                                             agglomerateToSegments)
-      agglomerateToPositions <- openZarrArrayCached(agglomerateFileKey, "agglomerate_to_positions")
-      position <- agglomerateToPositions.readAsMultiArray(offset = Array(segmentIndex, 0), shape = Array(3, 1))
-    } yield Vec3Int(position.getInt(0), position.getInt(1), position.getInt(2))
+  def positionForSegmentId(agglomerateFileKey: AgglomerateFileKey,
+                           segmentId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Vec3Int] = ???
 
   private def binarySearchForSegment(
       rangeStart: Long,
       rangeEnd: Long,
       segmentId: Long,
-      agglomerateToSegments: DatasetArray)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Long] =
-    if (rangeStart > rangeEnd) Fox.failure("Could not find segmentId in agglomerate file")
-    else {
-      val middle = rangeStart + (rangeEnd - rangeStart) / 2
-      for {
-        segmentIdAtMiddleMA <- agglomerateToSegments.readAsMultiArray(offset = middle, shape = 1)
-        segmentIdAdMiddleArray: Array[Long] <- MultiArrayUtils.toLongArray(segmentIdAtMiddleMA).toFox
-        segmentIdAtMiddle = segmentIdAdMiddleArray(0)
-        segmentIndex <- if (segmentIdAtMiddle == segmentId)
-          Fox.successful(middle)
-        else if (segmentIdAtMiddle < segmentId) {
-          binarySearchForSegment(middle + 1L, rangeEnd, segmentId, agglomerateToSegments)
-        } else binarySearchForSegment(rangeStart, middle - 1L, segmentId, agglomerateToSegments)
-      } yield segmentIndex
-    }
+      agglomerateToSegments: DatasetArray)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Long] = ???
 }
 
 class Hdf5AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala
index 9c24db0cb66..8330a386eb3 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala
@@ -159,7 +159,7 @@ class VolumeTracingZarrStreamingController @Inject()(
           chunks = Array(channels, cubeLength, cubeLength, cubeLength)
 
           zarrHeader = ZarrHeader(zarr_format = 2,
-                                  shape = shape.map(_.toLong),
+                                  shape = shape,
                                   chunks = chunks,
                                   compressor = compressor,
                                   dtype = dtype,
@@ -188,11 +188,11 @@ class VolumeTracingZarrStreamingController @Inject()(
             zarr_format = 3,
             node_type = "array",
             // channel, additional axes, XYZ
-            shape = (Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ Array(
+            shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ Array(
               (tracing.boundingBox.width + tracing.boundingBox.topLeft.x) / magParsed.x,
               (tracing.boundingBox.height + tracing.boundingBox.topLeft.y) / magParsed.y,
               (tracing.boundingBox.depth + tracing.boundingBox.topLeft.z) / magParsed.z
-            )).map(_.toLong),
+            ),
             data_type = Left(tracing.elementClass.toString),
             chunk_grid = Left(
               ChunkGridSpecification(

From a9f63c8ddf200d9978d6d37372a5079c5b5f9c42 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 28 May 2025 13:31:05 +0200
Subject: [PATCH 21/22] undo more stuff

---
 .../datastore/DataStoreModule.scala           |  1 -
 .../datastore/controllers/Application.scala   |  7 +++---
 .../DatasetArrayBucketProvider.scala          |  2 +-
 .../datareaders/MultiArrayUtils.scala         | 19 +---------------
 .../services/BinaryDataService.scala          | 22 ++-----------------
 5 files changed, 7 insertions(+), 44 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala
index 4b1ee3c06a2..7a2bd9b28ba 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala
@@ -28,7 +28,6 @@ class DataStoreModule extends AbstractModule {
     bind(classOf[BinaryDataServiceHolder]).asEagerSingleton()
     bind(classOf[MappingService]).asEagerSingleton()
     bind(classOf[AgglomerateService]).asEagerSingleton()
-    bind(classOf[ZarrAgglomerateService]).asEagerSingleton()
     bind(classOf[AdHocMeshServiceHolder]).asEagerSingleton()
     bind(classOf[ApplicationHealthService]).asEagerSingleton()
     bind(classOf[DSDatasetErrorLoggingService]).asEagerSingleton()
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
index 90e222df1be..693b917d7dd 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
@@ -4,7 +4,7 @@ import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.helpers.NativeBucketScanner
 import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
-import com.scalableminds.webknossos.datastore.services.{ApplicationHealthService, ZarrAgglomerateService}
+import com.scalableminds.webknossos.datastore.services.ApplicationHealthService
 import com.scalableminds.webknossos.datastore.storage.DataStoreRedisStore
 import net.liftweb.common.Box.tryo
 
@@ -13,9 +13,8 @@ import play.api.mvc.{Action, AnyContent}
 
 import scala.concurrent.ExecutionContext
 
-class Application @Inject()(redisClient: DataStoreRedisStore,
-                            applicationHealthService: ApplicationHealthService,
-                            agglomerateService: ZarrAgglomerateService)(implicit ec: ExecutionContext)
+class Application @Inject()(redisClient: DataStoreRedisStore, applicationHealthService: ApplicationHealthService)(
+    implicit ec: ExecutionContext)
     extends Controller {
 
   override def allowRemoteOrigin: Boolean = true
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
index 86f51996b3d..0c78fa32c7c 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala
@@ -38,8 +38,8 @@ class DatasetArrayBucketProvider(dataLayer: DataLayer,
       datasetArray <- datasetArrayCache.getOrLoad(readInstruction.bucket.mag,
                                                   _ => openDatasetArrayWithTimeLogging(readInstruction))
       bucket = readInstruction.bucket
-      offset = Vec3Int(bucket.topLeft.voxelXInMag, bucket.topLeft.voxelYInMag, bucket.topLeft.voxelZInMag)
       shape = Vec3Int.full(bucket.bucketLength)
+      offset = Vec3Int(bucket.topLeft.voxelXInMag, bucket.topLeft.voxelYInMag, bucket.topLeft.voxelZInMag)
       bucketData <- datasetArray.readBytesWithAdditionalCoordinates(shape,
                                                                     offset,
                                                                     bucket.additionalCoordinates,
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
index 86e19397b85..f1af69b890c 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/MultiArrayUtils.scala
@@ -2,7 +2,7 @@ package com.scalableminds.webknossos.datastore.datareaders
 
 import ArrayDataType.ArrayDataType
 import com.typesafe.scalalogging.LazyLogging
-import net.liftweb.common.{Box, Failure, Full}
+import net.liftweb.common.Box
 import net.liftweb.common.Box.tryo
 import ucar.ma2.{IndexIterator, InvalidRangeException, Range, Array => MultiArray, DataType => MADataType}
 
@@ -45,23 +45,6 @@ object MultiArrayUtils extends LazyLogging {
     }
   }
 
-  def createEmpty(rank: Int): MultiArray =
-    MultiArray.factory(MADataType.FLOAT, Array.fill(rank)(0))
-
-  def toLongArray(multiArray: MultiArray): Box[Array[Long]] =
-    multiArray.getDataType match {
-      case MADataType.LONG | MADataType.ULONG =>
-        Full(multiArray.getStorage.asInstanceOf[Array[Long]])
-      case MADataType.INT =>
-        Full(multiArray.getStorage.asInstanceOf[Array[Int]].map(_.toLong))
-      case MADataType.UINT =>
-        Full(multiArray.getStorage.asInstanceOf[Array[Int]].map { signed =>
-          if (signed >= 0) signed.toLong else signed.toLong + Int.MaxValue.toLong + Int.MaxValue.toLong + 2L
-        })
-      case _ =>
-        Failure("Cannot convert MultiArray to LongArray: unsupported data type.")
-    }
-
   /**
     * Offset describes the displacement between source and target array.<br/>
     * <br/>
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
index 58d4dd006e1..3852aa9fd49 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
@@ -136,26 +136,8 @@ class BinaryDataService(val dataBaseDir: Path,
     Full(outputArray)
   }
 
-  private def convertAccordingToRequest(request: DataServiceDataRequest, inputArray: Array[Byte])(
-      implicit tc: TokenContext): Fox[Array[Byte]] =
-    for {
-      clippedData <- convertIfNecessary(
-        !request.cuboid.toMag1BoundingBox.isFullyContainedIn(request.dataLayer.boundingBox),
-        inputArray,
-        data => clipToLayerBoundingBox(request)(data).toFox,
-        request
-      )
-      mappedDataFox <- agglomerateServiceOpt.map { agglomerateService =>
-        convertIfNecessary(
-          request.settings.appliedAgglomerate.isDefined && request.dataLayer.category == Category.segmentation && request.cuboid.mag.maxDim <= MaxMagForAgglomerateMapping,
-          clippedData,
-          data => agglomerateService.applyAgglomerate(request)(data),
-          request
-        )
-      }.toFox.fillEmpty(Fox.successful(clippedData)) ?~> "Failed to apply agglomerate mapping"
-      mappedData <- mappedDataFox
-      resultData <- convertIfNecessary(request.settings.halfByte, mappedData, convertToHalfByte, request)
-    } yield resultData
+  private def convertAccordingToRequest(request: DataServiceDataRequest, inputArray: Array[Byte]): Fox[Array[Byte]] =
+    Fox.successful(inputArray)
 
   def handleDataRequests(requests: List[DataServiceDataRequest])(
       implicit tc: TokenContext): Fox[(Array[Byte], List[Int])] = {

From 8a806e583ac482a93c341cdaa493290a297b5d8f Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 28 May 2025 13:33:06 +0200
Subject: [PATCH 22/22] more

---
 .../controllers/DataSourceController.scala    |  83 ++-
 .../services/AgglomerateService.scala         | 535 +++++++-----------
 .../services/SegmentIndexFileService.scala    |  33 +-
 .../services/mesh/AdHocMeshService.scala      |  12 +-
 .../services/mesh/MeshMappingHelper.scala     |  42 +-
 5 files changed, 292 insertions(+), 413 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
index a8ae4237f45..c6f84eaf0e0 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
@@ -277,11 +277,9 @@ class DataSourceController @Inject()(
       UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
       for {
         agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-        skeleton <- agglomerateService.generateSkeleton(organizationId,
-                                                        datasetDirectoryName,
-                                                        dataLayerName,
-                                                        mappingName,
-                                                        agglomerateId) ?~> "agglomerateSkeleton.failed"
+        skeleton <- agglomerateService
+          .generateSkeleton(organizationId, datasetDirectoryName, dataLayerName, mappingName, agglomerateId)
+          .toFox ?~> "agglomerateSkeleton.failed"
       } yield Ok(skeleton.toByteArray).as(protobufMimeType)
     }
   }
@@ -297,9 +295,11 @@ class DataSourceController @Inject()(
       UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
       for {
         agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-        agglomerateGraph <- agglomerateService.generateAgglomerateGraph(
-          AgglomerateFileKey(organizationId, datasetDirectoryName, dataLayerName, mappingName),
-          agglomerateId) ?~> "agglomerateGraph.failed"
+        agglomerateGraph <- agglomerateService
+          .generateAgglomerateGraph(
+            AgglomerateFileKey(organizationId, datasetDirectoryName, dataLayerName, mappingName),
+            agglomerateId)
+          .toFox ?~> "agglomerateGraph.failed"
       } yield Ok(agglomerateGraph.toByteArray).as(protobufMimeType)
     }
   }
@@ -315,9 +315,10 @@ class DataSourceController @Inject()(
       UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
       for {
         agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-        position <- agglomerateService.positionForSegmentId(
-          AgglomerateFileKey(organizationId, datasetDirectoryName, dataLayerName, mappingName),
-          segmentId) ?~> "getSegmentPositionFromAgglomerateFile.failed"
+        position <- agglomerateService
+          .positionForSegmentId(AgglomerateFileKey(organizationId, datasetDirectoryName, dataLayerName, mappingName),
+                                segmentId)
+          .toFox ?~> "getSegmentPositionFromAgglomerateFile.failed"
       } yield Ok(Json.toJson(position))
     }
   }
@@ -332,14 +333,16 @@ class DataSourceController @Inject()(
       UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
       for {
         agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-        largestAgglomerateId: Long <- agglomerateService.largestAgglomerateId(
-          AgglomerateFileKey(
-            organizationId,
-            datasetDirectoryName,
-            dataLayerName,
-            mappingName
+        largestAgglomerateId: Long <- agglomerateService
+          .largestAgglomerateId(
+            AgglomerateFileKey(
+              organizationId,
+              datasetDirectoryName,
+              dataLayerName,
+              mappingName
+            )
           )
-        )
+          .toFox
       } yield Ok(Json.toJson(largestAgglomerateId))
     }
   }
@@ -354,19 +357,45 @@ class DataSourceController @Inject()(
       UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
       for {
         agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-        agglomerateIds: Seq[Long] <- agglomerateService.agglomerateIdsForSegmentIds(
-          AgglomerateFileKey(
-            organizationId,
-            datasetDirectoryName,
-            dataLayerName,
-            mappingName
-          ),
-          request.body.items
-        )
+        agglomerateIds: Seq[Long] <- agglomerateService
+          .agglomerateIdsForSegmentIds(
+            AgglomerateFileKey(
+              organizationId,
+              datasetDirectoryName,
+              dataLayerName,
+              mappingName
+            ),
+            request.body.items
+          )
+          .toFox
       } yield Ok(ListOfLong(agglomerateIds).toByteArray)
     }
   }
 
+  def agglomerateIdsForAllSegmentIds(
+      organizationId: String,
+      datasetDirectoryName: String,
+      dataLayerName: String,
+      mappingName: String
+  ): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request =>
+    accessTokenService.validateAccessFromTokenContext(
+      UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
+      for {
+        agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
+        agglomerateIds: Array[Long] <- agglomerateService
+          .agglomerateIdsForAllSegmentIds(
+            AgglomerateFileKey(
+              organizationId,
+              datasetDirectoryName,
+              dataLayerName,
+              mappingName
+            )
+          )
+          .toFox
+      } yield Ok(Json.toJson(agglomerateIds))
+    }
+  }
+
   def update(organizationId: String, datasetDirectoryName: String): Action[DataSource] =
     Action.async(validateJson[DataSource]) { implicit request =>
       accessTokenService.validateAccessFromTokenContext(
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 19b39dd10d8..ed970194131 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -1,142 +1,36 @@
 package com.scalableminds.webknossos.datastore.services
 
 import ch.systemsx.cisd.hdf5._
-import com.scalableminds.util.accesscontext.TokenContext
-import com.scalableminds.util.cache.AlfuCache
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.io.PathUtils
 import com.scalableminds.util.time.Instant
-import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.AgglomerateGraph.{AgglomerateEdge, AgglomerateGraph}
 import com.scalableminds.webknossos.datastore.DataStoreConfig
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, SkeletonTracing, Tree, TreeTypeProto}
-import com.scalableminds.webknossos.datastore.datareaders.{DatasetArray, MultiArrayUtils}
-import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3Array
 import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto
-import com.scalableminds.webknossos.datastore.helpers.{NativeBucketScanner, NodeDefaults, SkeletonTracingDefaults}
-import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, ElementClass}
+import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, SkeletonTracingDefaults}
+import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
 import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
 import com.scalableminds.webknossos.datastore.storage._
 import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.{Box, Failure, Full}
 import net.liftweb.common.Box.tryo
 import org.apache.commons.io.FilenameUtils
-import ucar.ma2.{Array => MultiArray}
 
-import java.net.URI
 import java.nio._
 import java.nio.file.{Files, Paths}
 import javax.inject.Inject
 import scala.annotation.tailrec
 import scala.collection.compat.immutable.ArraySeq
-import scala.concurrent.ExecutionContext
 import scala.concurrent.duration.DurationInt
 
-class ZarrAgglomerateService @Inject()(config: DataStoreConfig, dataVaultService: DataVaultService)
-    extends DataConverter
-    with LazyLogging {
-  private val dataBaseDir = Paths.get(config.Datastore.baseDirectory)
-  private val agglomerateDir = "agglomerates"
-
-  // TODO clear on dataset reload
-  private lazy val openArraysCache = AlfuCache[(AgglomerateFileKey, String), DatasetArray]()
-
-  // TODO unify with existing chunkContentsCache from binaryDataService?
-  private lazy val sharedChunkContentsCache: AlfuCache[String, MultiArray] = {
-    // Used by DatasetArray-based datasets. Measure item weight in kilobytes because the weigher can only return int, not long
-
-    val maxSizeKiloBytes = Math.floor(config.Datastore.Cache.ImageArrayChunks.maxSizeBytes.toDouble / 1000.0).toInt
-
-    def cacheWeight(key: String, arrayBox: Box[MultiArray]): Int =
-      arrayBox match {
-        case Full(array) =>
-          (array.getSizeBytes / 1000L).toInt
-        case _ => 0
-      }
-
-    AlfuCache(maxSizeKiloBytes, weighFn = Some(cacheWeight))
-  }
-
-  protected lazy val bucketScanner = new NativeBucketScanner()
-
-  private def mapSingleSegment(segmentToAgglomerate: DatasetArray, segmentId: Long)(implicit ec: ExecutionContext,
-                                                                                    tc: TokenContext): Fox[Long] = ???
-
-  private def openZarrArrayCached(agglomerateFileKey: AgglomerateFileKey,
-                                  zarrArrayName: String)(implicit ec: ExecutionContext, tc: TokenContext) =
-    openArraysCache.getOrLoad((agglomerateFileKey, zarrArrayName),
-                              _ => openZarrArray(agglomerateFileKey, zarrArrayName))
-
-  private def openZarrArray(agglomerateFileKey: AgglomerateFileKey, zarrArrayName: String)(
-      implicit ec: ExecutionContext,
-      tc: TokenContext): Fox[DatasetArray] = {
-
-    val zarrGroupPath = agglomerateFileKey.zarrGroupPath(dataBaseDir, agglomerateDir).toAbsolutePath
-    for {
-      groupVaultPath <- dataVaultService.getVaultPath(RemoteSourceDescriptor(new URI(s"file://$zarrGroupPath"), None))
-      segmentToAgglomeratePath = groupVaultPath / zarrArrayName
-      zarrArray <- Zarr3Array.open(segmentToAgglomeratePath,
-                                   DataSourceId("zarr", "test"),
-                                   "layer",
-                                   None,
-                                   None,
-                                   None,
-                                   sharedChunkContentsCache)
-    } yield zarrArray
-  }
-
-  def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(implicit ec: ExecutionContext,
-                                                                           tc: TokenContext): Fox[Array[Byte]] = ???
-
-  def generateSkeleton(organizationId: String,
-                       datasetDirectoryName: String,
-                       dataLayerName: String,
-                       mappingName: String,
-                       agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[SkeletonTracing] = ???
-
-  def largestAgglomerateId(agglomerateFileKey: AgglomerateFileKey)(implicit ec: ExecutionContext,
-                                                                   tc: TokenContext): Fox[Long] = ???
-
-  def generateAgglomerateGraph(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long)(
-      implicit ec: ExecutionContext,
-      tc: TokenContext): Fox[AgglomerateGraph] = ???
-
-  def segmentIdsForAgglomerateId(agglomerateFileKey: AgglomerateFileKey,
-                                 agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Seq[Long]] =
-    ???
-
-  def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: Seq[Long])(
-      implicit ec: ExecutionContext,
-      tc: TokenContext): Fox[Seq[Long]] = ???
-
-  def positionForSegmentId(agglomerateFileKey: AgglomerateFileKey,
-                           segmentId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Vec3Int] = ???
-
-  private def binarySearchForSegment(
-      rangeStart: Long,
-      rangeEnd: Long,
-      segmentId: Long,
-      agglomerateToSegments: DatasetArray)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Long] = ???
-}
-
-class Hdf5AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
-  // TODO
-}
-
-class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateService: ZarrAgglomerateService)
-    extends DataConverter
-    with LazyLogging
-    with FoxImplicits {
+class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverter with LazyLogging {
   private val agglomerateDir = "agglomerates"
   private val agglomerateFileExtension = "hdf5"
   private val datasetName = "/segment_to_agglomerate"
   private val dataBaseDir = Paths.get(config.Datastore.baseDirectory)
   private val cumsumFileName = "cumsum.json"
 
-  // TODO remove
-  private val useZarr = false
-
-  // TODO clear on reload
   lazy val agglomerateFileCache = new AgglomerateFileCache(config.Datastore.Cache.AgglomerateFile.maxFileHandleEntries)
 
   def exploreAgglomerates(organizationId: String, datasetDirectoryName: String, dataLayerName: String): Set[String] = {
@@ -149,38 +43,11 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
         paths.map(path => FilenameUtils.removeExtension(path.getFileName.toString))
       }
       .toOption
-      .getOrElse(Nil) // TODO explore zarr agglomerates?
-      .toSet ++ Set(
-      "agglomerate_view_5",
-      "agglomerate_view_10",
-      "agglomerate_view_15",
-      "agglomerate_view_20",
-      "agglomerate_view_25",
-      "agglomerate_view_30",
-      "agglomerate_view_35",
-      "agglomerate_view_40",
-      "agglomerate_view_45",
-      "agglomerate_view_50",
-      "agglomerate_view_55",
-      "agglomerate_view_60",
-      "agglomerate_view_65",
-      "agglomerate_view_70",
-      "agglomerate_view_75",
-      "agglomerate_view_80",
-      "agglomerate_view_85",
-      "agglomerate_view_90",
-      "agglomerate_view_95",
-      "agglomerate_view_100"
-    )
+      .getOrElse(Nil)
+      .toSet
   }
 
-  def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(implicit ec: ExecutionContext,
-                                                                           tc: TokenContext): Fox[Array[Byte]] =
-    if (useZarr) {
-      zarrAgglomerateService.applyAgglomerate(request)(data)
-    } else applyAgglomerateHdf5(request)(data).toFox
-
-  private def applyAgglomerateHdf5(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] = tryo {
+  def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte]): Box[Array[Byte]] = tryo {
 
     val agglomerateFileKey = AgglomerateFileKey.fromDataRequest(request)
 
@@ -235,7 +102,7 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
     // We don't need to differentiate between the data types because the underlying library does the conversion for us
     reader.uint64().readArrayBlockWithOffset(hdf5Dataset, blockSize.toInt, segmentId)
 
-  // This uses the datasetName, which allows us to call it on the same hdf file in parallel.
+  // This uses the datasetDirectoryName, which allows us to call it on the same hdf file in parallel.
   private def readHDF(reader: IHDF5Reader, segmentId: Long, blockSize: Long) =
     reader.uint64().readArrayBlockWithOffset(datasetName, blockSize.toInt, segmentId)
 
@@ -274,174 +141,162 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
                        datasetDirectoryName: String,
                        dataLayerName: String,
                        mappingName: String,
-                       agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[SkeletonTracing] =
-    if (useZarr) {
-      zarrAgglomerateService.generateSkeleton(organizationId,
-                                              datasetDirectoryName,
-                                              dataLayerName,
-                                              mappingName,
-                                              agglomerateId)
-    } else {
-      (try {
-        val before = Instant.now
-        val hdfFile =
-          dataBaseDir
-            .resolve(organizationId)
-            .resolve(datasetDirectoryName)
-            .resolve(dataLayerName)
-            .resolve(agglomerateDir)
-            .resolve(s"$mappingName.$agglomerateFileExtension")
-            .toFile
-
-        val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
-        val positionsRange: Array[Long] =
-          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
-        val edgesRange: Array[Long] =
-          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
-
-        val nodeCount = positionsRange(1) - positionsRange(0)
-        val edgeCount = edgesRange(1) - edgesRange(0)
-        val edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
-        if (nodeCount > edgeLimit) {
-          throw new Exception(s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)")
-        }
-        if (edgeCount > edgeLimit) {
-          throw new Exception(s"Agglomerate has too many edges ($edgeCount > $edgeLimit)")
+                       agglomerateId: Long): Box[SkeletonTracing] =
+    try {
+      val before = Instant.now
+      val hdfFile =
+        dataBaseDir
+          .resolve(organizationId)
+          .resolve(datasetDirectoryName)
+          .resolve(dataLayerName)
+          .resolve(agglomerateDir)
+          .resolve(s"$mappingName.$agglomerateFileExtension")
+          .toFile
+
+      val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+      val positionsRange: Array[Long] =
+        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
+      val edgesRange: Array[Long] =
+        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
+
+      val nodeCount = positionsRange(1) - positionsRange(0)
+      val edgeCount = edgesRange(1) - edgesRange(0)
+      val edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
+      if (nodeCount > edgeLimit) {
+        throw new Exception(s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)")
+      }
+      if (edgeCount > edgeLimit) {
+        throw new Exception(s"Agglomerate has too many edges ($edgeCount > $edgeLimit)")
+      }
+      val positions: Array[Array[Long]] =
+        if (nodeCount == 0L) {
+          Array.empty[Array[Long]]
+        } else {
+          reader
+            .uint64()
+            .readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
         }
-        val positions: Array[Array[Long]] =
-          if (nodeCount == 0L) {
-            Array.empty[Array[Long]]
-          } else {
-            reader
-              .uint64()
-              .readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
-          }
-        val edges: Array[Array[Long]] = {
-          if (edgeCount == 0L) {
-            Array.empty[Array[Long]]
-          } else {
-            reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
-          }
+      val edges: Array[Array[Long]] = {
+        if (edgeCount == 0L) {
+          Array.empty[Array[Long]]
+        } else {
+          reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
         }
+      }
 
-        val nodeIdStartAtOneOffset = 1
-
-        val nodes = positions.zipWithIndex.map {
-          case (pos, idx) =>
-            NodeDefaults.createInstance.copy(
-              id = idx + nodeIdStartAtOneOffset,
-              position = Vec3IntProto(pos(0).toInt, pos(1).toInt, pos(2).toInt)
-            )
-        }
+      val nodeIdStartAtOneOffset = 1
 
-        val skeletonEdges = edges.map { e =>
-          Edge(source = e(0).toInt + nodeIdStartAtOneOffset, target = e(1).toInt + nodeIdStartAtOneOffset)
-        }
+      val nodes = positions.zipWithIndex.map {
+        case (pos, idx) =>
+          NodeDefaults.createInstance.copy(
+            id = idx + nodeIdStartAtOneOffset,
+            position = Vec3IntProto(pos(0).toInt, pos(1).toInt, pos(2).toInt)
+          )
+      }
 
-        val trees = Seq(
-          Tree(
-            treeId = math.abs(agglomerateId.toInt), // used only to deterministically select tree color
-            createdTimestamp = System.currentTimeMillis(),
-            // unsafeWrapArray is fine, because the underlying arrays are never mutated
-            nodes = ArraySeq.unsafeWrapArray(nodes),
-            edges = ArraySeq.unsafeWrapArray(skeletonEdges),
-            name = s"agglomerate $agglomerateId ($mappingName)",
-            `type` = Some(TreeTypeProto.AGGLOMERATE)
-          ))
-
-        val skeleton = SkeletonTracingDefaults.createInstance.copy(
-          datasetName = datasetDirectoryName,
-          trees = trees
-        )
-
-        if (Instant.since(before) > (100 milliseconds)) {
-          Instant.logSince(
-            before,
-            s"Generating skeleton from agglomerate file with ${skeletonEdges.length} edges, ${nodes.length} nodes",
-            logger)
-        }
+      val skeletonEdges = edges.map { e =>
+        Edge(source = e(0).toInt + nodeIdStartAtOneOffset, target = e(1).toInt + nodeIdStartAtOneOffset)
+      }
 
-        Full(skeleton)
-      } catch {
-        case e: Exception => Failure(e.getMessage)
-      }).toFox
-    }
+      val trees = Seq(
+        Tree(
+          treeId = math.abs(agglomerateId.toInt), // used only to deterministically select tree color
+          createdTimestamp = System.currentTimeMillis(),
+          // unsafeWrapArray is fine, because the underlying arrays are never mutated
+          nodes = ArraySeq.unsafeWrapArray(nodes),
+          edges = ArraySeq.unsafeWrapArray(skeletonEdges),
+          name = s"agglomerate $agglomerateId ($mappingName)",
+          `type` = Some(TreeTypeProto.AGGLOMERATE)
+        ))
+
+      val skeleton = SkeletonTracingDefaults.createInstance.copy(
+        datasetName = datasetDirectoryName,
+        trees = trees
+      )
+
+      if (Instant.since(before) > (100 milliseconds)) {
+        Instant.logSince(
+          before,
+          s"Generating skeleton from agglomerate file with ${skeletonEdges.length} edges, ${nodes.length} nodes",
+          logger)
+      }
 
-  def largestAgglomerateId(agglomerateFileKey: AgglomerateFileKey)(implicit ec: ExecutionContext,
-                                                                   tc: TokenContext): Fox[Long] =
-    if (useZarr) zarrAgglomerateService.largestAgglomerateId(agglomerateFileKey)
-    else {
-      val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
-      tryo {
-        val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
-        reader.`object`().getNumberOfElements("/agglomerate_to_segments_offsets") - 1L
-      }.toFox
+      Full(skeleton)
+    } catch {
+      case e: Exception => Failure(e.getMessage)
     }
 
-  def segmentIdsForAgglomerateId(agglomerateFileKey: AgglomerateFileKey,
-                                 agglomerateId: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Seq[Long]] =
-    if (useZarr)
-      zarrAgglomerateService.segmentIdsForAgglomerateId(agglomerateFileKey, agglomerateId)
-    else {
-      val hdfFile =
-        dataBaseDir
-          .resolve(agglomerateFileKey.organizationId)
-          .resolve(agglomerateFileKey.datasetDirectoryName)
-          .resolve(agglomerateFileKey.layerName)
-          .resolve(agglomerateDir)
-          .resolve(s"${agglomerateFileKey.mappingName}.$agglomerateFileExtension")
-          .toFile
+  def largestAgglomerateId(agglomerateFileKey: AgglomerateFileKey): Box[Long] = {
+    val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
 
-      tryo {
-        val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
-        val positionsRange: Array[Long] =
-          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
-
-        val segmentCount = positionsRange(1) - positionsRange(0)
-        val segmentIds: Array[Long] =
-          if (segmentCount == 0) Array.empty[Long]
-          else {
-            reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", segmentCount.toInt, positionsRange(0))
-          }
-        segmentIds.toSeq
-      }.toFox
+    tryo {
+      val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+      reader.`object`().getNumberOfElements("/agglomerate_to_segments_offsets") - 1L
     }
+  }
 
-  def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: Seq[Long])(
-      implicit ec: ExecutionContext,
-      tc: TokenContext): Fox[Seq[Long]] =
-    if (useZarr) {
-      zarrAgglomerateService.agglomerateIdsForSegmentIds(agglomerateFileKey, segmentIds)
-    } else {
-      val cachedAgglomerateFile = agglomerateFileCache.withCache(agglomerateFileKey)(initHDFReader)
-      tryo {
-        val agglomerateIds = segmentIds.map { segmentId: Long =>
-          cachedAgglomerateFile.agglomerateIdCache.withCache(segmentId,
-                                                             cachedAgglomerateFile.reader,
-                                                             cachedAgglomerateFile.dataset)(readHDF)
+  def segmentIdsForAgglomerateId(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long): Box[List[Long]] = {
+    val hdfFile =
+      dataBaseDir
+        .resolve(agglomerateFileKey.organizationId)
+        .resolve(agglomerateFileKey.datasetDirectoryName)
+        .resolve(agglomerateFileKey.layerName)
+        .resolve(agglomerateDir)
+        .resolve(s"${agglomerateFileKey.mappingName}.$agglomerateFileExtension")
+        .toFile
+
+    tryo {
+      val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+      val positionsRange: Array[Long] =
+        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
+
+      val segmentCount = positionsRange(1) - positionsRange(0)
+      val segmentIds: Array[Long] =
+        if (segmentCount == 0) Array.empty[Long]
+        else {
+          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", segmentCount.toInt, positionsRange(0))
         }
-        cachedAgglomerateFile.finishAccess()
-        agglomerateIds
-      }.toFox
+      segmentIds.toList
     }
+  }
 
-  def positionForSegmentId(agglomerateFileKey: AgglomerateFileKey, segmentId: Long)(implicit ec: ExecutionContext,
-                                                                                    tc: TokenContext): Fox[Vec3Int] =
-    if (useZarr) zarrAgglomerateService.positionForSegmentId(agglomerateFileKey, segmentId)
-    else {
-      val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
-      val reader: IHDF5Reader = HDF5FactoryProvider.get.openForReading(hdfFile)
-      (for {
-        agglomerateIdArr: Array[Long] <- tryo(
-          reader.uint64().readArrayBlockWithOffset("/segment_to_agglomerate", 1, segmentId))
-        agglomerateId = agglomerateIdArr(0)
-        segmentsRange: Array[Long] <- tryo(
-          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId))
-        segmentIndex <- binarySearchForSegment(segmentsRange(0), segmentsRange(1), segmentId, reader)
-        position <- tryo(
-          reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_positions", 1, 3, segmentIndex, 0)(0))
-      } yield Vec3Int(position(0).toInt, position(1).toInt, position(2).toInt)).toFox
+  def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: Seq[Long]): Box[Seq[Long]] = {
+    val cachedAgglomerateFile = agglomerateFileCache.withCache(agglomerateFileKey)(initHDFReader)
+
+    tryo {
+      val agglomerateIds = segmentIds.map { segmentId: Long =>
+        cachedAgglomerateFile.agglomerateIdCache.withCache(segmentId,
+                                                           cachedAgglomerateFile.reader,
+                                                           cachedAgglomerateFile.dataset)(readHDF)
+      }
+      cachedAgglomerateFile.finishAccess()
+      agglomerateIds
+    }
+
+  }
+
+  def agglomerateIdsForAllSegmentIds(agglomerateFileKey: AgglomerateFileKey): Box[Array[Long]] = {
+    val file = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
+    tryo {
+      val reader = HDF5FactoryProvider.get.openForReading(file)
+      val agglomerateIds: Array[Long] = reader.uint64().readArray("/segment_to_agglomerate")
+      agglomerateIds
     }
+  }
+
+  def positionForSegmentId(agglomerateFileKey: AgglomerateFileKey, segmentId: Long): Box[Vec3Int] = {
+    val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
+    val reader: IHDF5Reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+    for {
+      agglomerateIdArr: Array[Long] <- tryo(
+        reader.uint64().readArrayBlockWithOffset("/segment_to_agglomerate", 1, segmentId))
+      agglomerateId = agglomerateIdArr(0)
+      segmentsRange: Array[Long] <- tryo(
+        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId))
+      segmentIndex <- binarySearchForSegment(segmentsRange(0), segmentsRange(1), segmentId, reader)
+      position <- tryo(reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_positions", 1, 3, segmentIndex, 0)(0))
+    } yield Vec3Int(position(0).toInt, position(1).toInt, position(2).toInt)
+  }
 
   @tailrec
   private def binarySearchForSegment(rangeStart: Long,
@@ -457,60 +312,54 @@ class AgglomerateService @Inject()(config: DataStoreConfig, zarrAgglomerateServi
       else binarySearchForSegment(rangeStart, middle - 1L, segmentId, reader)
     }
 
-  def generateAgglomerateGraph(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long)(
-      implicit ec: ExecutionContext,
-      tc: TokenContext): Fox[AgglomerateGraph] =
-    if (useZarr)
-      zarrAgglomerateService.generateAgglomerateGraph(agglomerateFileKey, agglomerateId)
-    else {
-      tryo {
-        val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
+  def generateAgglomerateGraph(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long): Box[AgglomerateGraph] =
+    tryo {
+      val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
 
-        val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+      val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
 
-        val positionsRange: Array[Long] =
-          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
-        val edgesRange: Array[Long] =
-          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
+      val positionsRange: Array[Long] =
+        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
+      val edgesRange: Array[Long] =
+        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
 
-        val nodeCount = positionsRange(1) - positionsRange(0)
-        val edgeCount = edgesRange(1) - edgesRange(0)
-        val edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
-        if (nodeCount > edgeLimit) {
-          throw new Exception(s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)")
-        }
-        if (edgeCount > edgeLimit) {
-          throw new Exception(s"Agglomerate has too many edges ($edgeCount > $edgeLimit)")
-        }
-        val segmentIds: Array[Long] =
-          if (nodeCount == 0L) Array[Long]()
-          else
-            reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", nodeCount.toInt, positionsRange(0))
-        val positions: Array[Array[Long]] =
-          if (nodeCount == 0L) Array[Array[Long]]()
-          else
-            reader
-              .uint64()
-              .readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
-        val edges: Array[Array[Long]] =
-          if (edgeCount == 0L) Array[Array[Long]]()
-          else
-            reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
-        val affinities: Array[Float] =
-          if (edgeCount == 0L) Array[Float]()
-          else
-            reader.float32().readArrayBlockWithOffset("/agglomerate_to_affinities", edgeCount.toInt, edgesRange(0))
-
-        AgglomerateGraph(
-          // unsafeWrapArray is fine, because the underlying arrays are never mutated
-          segments = ArraySeq.unsafeWrapArray(segmentIds),
-          edges = ArraySeq.unsafeWrapArray(
-            edges.map(e => AgglomerateEdge(source = segmentIds(e(0).toInt), target = segmentIds(e(1).toInt)))),
-          positions =
-            ArraySeq.unsafeWrapArray(positions.map(pos => Vec3IntProto(pos(0).toInt, pos(1).toInt, pos(2).toInt))),
-          affinities = ArraySeq.unsafeWrapArray(affinities)
-        )
-      }.toFox
+      val nodeCount = positionsRange(1) - positionsRange(0)
+      val edgeCount = edgesRange(1) - edgesRange(0)
+      val edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
+      if (nodeCount > edgeLimit) {
+        throw new Exception(s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)")
+      }
+      if (edgeCount > edgeLimit) {
+        throw new Exception(s"Agglomerate has too many edges ($edgeCount > $edgeLimit)")
+      }
+      val segmentIds: Array[Long] =
+        if (nodeCount == 0L) Array[Long]()
+        else
+          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", nodeCount.toInt, positionsRange(0))
+      val positions: Array[Array[Long]] =
+        if (nodeCount == 0L) Array[Array[Long]]()
+        else
+          reader
+            .uint64()
+            .readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
+      val edges: Array[Array[Long]] =
+        if (edgeCount == 0L) Array[Array[Long]]()
+        else
+          reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
+      val affinities: Array[Float] =
+        if (edgeCount == 0L) Array[Float]()
+        else
+          reader.float32().readArrayBlockWithOffset("/agglomerate_to_affinities", edgeCount.toInt, edgesRange(0))
+
+      AgglomerateGraph(
+        // unsafeWrapArray is fine, because the underlying arrays are never mutated
+        segments = ArraySeq.unsafeWrapArray(segmentIds),
+        edges = ArraySeq.unsafeWrapArray(
+          edges.map(e => AgglomerateEdge(source = segmentIds(e(0).toInt), target = segmentIds(e(1).toInt)))),
+        positions =
+          ArraySeq.unsafeWrapArray(positions.map(pos => Vec3IntProto(pos(0).toInt, pos(1).toInt, pos(2).toInt))),
+        affinities = ArraySeq.unsafeWrapArray(affinities)
+      )
     }
 
 }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala
index 108d6b4c9ce..541b5a040d5 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala
@@ -185,11 +185,11 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig,
         ))
       bucketData <- binaryDataServiceHolder.binaryDataService.handleMultipleBucketRequests(bucketRequests)
     } yield (bucketData, dataLayer.elementClass)
-  private def getBucketPositions(organizationId: String,
-                                 datasetDirectoryName: String,
-                                 dataLayerName: String,
-                                 mappingName: Option[String])(segmentOrAgglomerateId: Long, mag: Vec3Int)(
-      implicit tc: TokenContext): Fox[Set[Vec3IntProto]] =
+  private def getBucketPositions(
+      organizationId: String,
+      datasetDirectoryName: String,
+      dataLayerName: String,
+      mappingName: Option[String])(segmentOrAgglomerateId: Long, mag: Vec3Int): Fox[Set[Vec3IntProto]] =
     for {
       segmentIds <- getSegmentIdsForAgglomerateIdIfNeeded(organizationId,
                                                           datasetDirectoryName,
@@ -212,12 +212,11 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig,
       bucketPositions = bucketPositionsInFileMag.map(_ / (mag / fileMag))
     } yield bucketPositions
 
-  private def getSegmentIdsForAgglomerateIdIfNeeded(
-      organizationId: String,
-      datasetDirectoryName: String,
-      dataLayerName: String,
-      segmentOrAgglomerateId: Long,
-      mappingNameOpt: Option[String])(implicit tc: TokenContext): Fox[Seq[Long]] =
+  private def getSegmentIdsForAgglomerateIdIfNeeded(organizationId: String,
+                                                    datasetDirectoryName: String,
+                                                    dataLayerName: String,
+                                                    segmentOrAgglomerateId: Long,
+                                                    mappingNameOpt: Option[String]): Fox[List[Long]] =
     // Editable mappings cannot happen here since those requests go to the tracingstore
     mappingNameOpt match {
       case Some(mappingName) =>
@@ -229,12 +228,14 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig,
             dataLayerName,
             mappingName
           )
-          largestAgglomerateId <- agglomerateService.largestAgglomerateId(agglomerateFileKey)
+          largestAgglomerateId <- agglomerateService.largestAgglomerateId(agglomerateFileKey).toFox
           segmentIds <- if (segmentOrAgglomerateId <= largestAgglomerateId) {
-            agglomerateService.segmentIdsForAgglomerateId(
-              agglomerateFileKey,
-              segmentOrAgglomerateId
-            )
+            agglomerateService
+              .segmentIdsForAgglomerateId(
+                agglomerateFileKey,
+                segmentOrAgglomerateId
+              )
+              .toFox
           } else
             Fox.successful(List.empty) // agglomerate id is outside of file range, was likely created during brushing
         } yield segmentIds
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala
index 6e711449df3..36e01971f44 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/AdHocMeshService.scala
@@ -14,7 +14,7 @@ import com.scalableminds.webknossos.datastore.models.requests.{
 import com.scalableminds.webknossos.datastore.services.mcubes.MarchingCubes
 import com.scalableminds.webknossos.datastore.services.{BinaryDataService, MappingService}
 import com.typesafe.scalalogging.LazyLogging
-import net.liftweb.common.{Box, Failure}
+import net.liftweb.common.{Box, Failure, Full}
 import org.apache.pekko.actor.{Actor, ActorRef, ActorSystem, Props}
 import org.apache.pekko.pattern.ask
 import org.apache.pekko.routing.RoundRobinPool
@@ -111,7 +111,7 @@ class AdHocMeshService(binaryDataService: BinaryDataService,
           Fox.successful(data)
       }
 
-    def applyAgglomerate(data: Array[Byte]): Fox[Array[Byte]] =
+    def applyAgglomerate(data: Array[Byte]): Box[Array[Byte]] =
       request.mapping match {
         case Some(_) =>
           request.mappingType match {
@@ -124,12 +124,12 @@ class AdHocMeshService(binaryDataService: BinaryDataService,
                   DataServiceRequestSettings(halfByte = false, request.mapping, None)
                 )
                 agglomerateService.applyAgglomerate(dataRequest)(data)
-              }.getOrElse(Fox.successful(data))
+              }.getOrElse(Full(data))
             case _ =>
-              Fox.successful(data)
+              Full(data)
           }
         case _ =>
-          Fox.successful(data)
+          Full(data)
       }
 
     def convertData(data: Array[Byte]): Array[T] = {
@@ -193,7 +193,7 @@ class AdHocMeshService(binaryDataService: BinaryDataService,
 
     for {
       data <- binaryDataService.handleDataRequest(dataRequest)
-      agglomerateMappedData <- applyAgglomerate(data) ?~> "failed to apply agglomerate for ad-hoc meshing"
+      agglomerateMappedData <- applyAgglomerate(data).toFox ?~> "failed to apply agglomerate for ad-hoc meshing"
       typedData = convertData(agglomerateMappedData)
       mappedData <- applyMapping(typedData)
       mappedSegmentId <- applyMapping(Array(typedSegmentId)).map(_.head)
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshMappingHelper.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshMappingHelper.scala
index 96a688e980c..ecee0011d7a 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshMappingHelper.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshMappingHelper.scala
@@ -27,7 +27,7 @@ trait MeshMappingHelper extends FoxImplicits {
       agglomerateId: Long,
       mappingNameForMeshFile: Option[String],
       omitMissing: Boolean // If true, failing lookups in the agglomerate file will just return empty list.
-  )(implicit ec: ExecutionContext, tc: TokenContext): Fox[Seq[Long]] =
+  )(implicit ec: ExecutionContext, tc: TokenContext): Fox[List[Long]] =
     (targetMappingName, editableMappingTracingId) match {
       case (None, None) =>
         // No mapping selected, assume id matches meshfile
@@ -40,17 +40,15 @@ trait MeshMappingHelper extends FoxImplicits {
         // assume agglomerate id, fetch oversegmentation segment ids for it
         for {
           agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-          segmentIdsBox <- agglomerateService
-            .segmentIdsForAgglomerateId(
-              AgglomerateFileKey(
-                organizationId,
-                datasetDirectoryName,
-                dataLayerName,
-                mappingName
-              ),
-              agglomerateId
-            )
-            .shiftBox
+          segmentIdsBox = agglomerateService.segmentIdsForAgglomerateId(
+            AgglomerateFileKey(
+              organizationId,
+              datasetDirectoryName,
+              dataLayerName,
+              mappingName
+            ),
+            agglomerateId
+          )
           segmentIds <- segmentIdsBox match {
             case Full(segmentIds) => Fox.successful(segmentIds)
             case _                => if (omitMissing) Fox.successful(List.empty) else segmentIdsBox.toFox
@@ -69,15 +67,17 @@ trait MeshMappingHelper extends FoxImplicits {
           else // the agglomerate id is not present in the editable mapping. Fetch its info from the base mapping.
             for {
               agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-              localSegmentIds <- agglomerateService.segmentIdsForAgglomerateId(
-                AgglomerateFileKey(
-                  organizationId,
-                  datasetDirectoryName,
-                  dataLayerName,
-                  mappingName
-                ),
-                agglomerateId
-              )
+              localSegmentIds <- agglomerateService
+                .segmentIdsForAgglomerateId(
+                  AgglomerateFileKey(
+                    organizationId,
+                    datasetDirectoryName,
+                    dataLayerName,
+                    mappingName
+                  ),
+                  agglomerateId
+                )
+                .toFox
             } yield localSegmentIds
         } yield segmentIds
       case _ => Fox.failure("Cannot determine segment ids for editable mapping without base mapping")