Skip to content

Commit f552a57

Browse files
daniel-werdaniel-wer
committed
Merge branch 'master' of github.com:scalableminds/webknossos into avoid-webgl-crashes
2 parents 3a7fc38 + 3a9cd8a commit f552a57

File tree

14 files changed

+3007
-385
lines changed

14 files changed

+3007
-385
lines changed

‎app/controllers/InitialDataController.scala‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package controllers
22

33
import play.silhouette.api.{LoginInfo, Silhouette}
44
import com.scalableminds.util.accesscontext.GlobalAccessContext
5+
import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int}
56
import com.scalableminds.util.objectid.ObjectId
67
import com.scalableminds.util.time.Instant
78
import com.scalableminds.util.tools.{Fox, FoxImplicits}
@@ -15,6 +16,17 @@ import models.task.{TaskType, TaskTypeDAO}
1516
import models.team._
1617
import models.user._
1718
import com.scalableminds.util.tools.Full
19+
import com.scalableminds.webknossos.datastore.dataformats.MagLocator
20+
import com.scalableminds.webknossos.datastore.helpers.UPath
21+
import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize}
22+
import com.scalableminds.webknossos.datastore.models.datasource.{
23+
DataFormat,
24+
DataSourceId,
25+
ElementClass,
26+
StaticColorLayer,
27+
StaticSegmentationLayer,
28+
UsableDataSource
29+
}
1830
import play.api.libs.json.{JsArray, Json}
1931
import utils.{StoreModules, WkConf}
2032

@@ -39,6 +51,8 @@ class InitialDataController @Inject()(initialDataService: InitialDataService, si
3951

4052
class InitialDataService @Inject()(userService: UserService,
4153
userDAO: UserDAO,
54+
datasetDAO: DatasetDAO,
55+
datasetLayerDAO: DatasetLayerDAO,
4256
multiUserDAO: MultiUserDAO,
4357
userExperiencesDAO: UserExperiencesDAO,
4458
taskTypeDAO: TaskTypeDAO,
@@ -156,6 +170,95 @@ Samplecountry
156170
Some("Works if model files are manually placed at binaryData/sample_organization/66544a56d20000af0e42ba0f/"),
157171
Some(AiModelCategory.em_neurons)
158172
)
173+
private val defaultDataSource = UsableDataSource(
174+
id = DataSourceId("l4_sample_remote", defaultOrganization._id),
175+
dataLayers = List(
176+
StaticColorLayer(
177+
name = "color",
178+
dataFormat = DataFormat.zarr3,
179+
boundingBox = BoundingBox(Vec3Int(3072, 3072, 512), 1024, 1024, 1024),
180+
elementClass = ElementClass.uint8,
181+
mags = List(
182+
MagLocator(
183+
mag = Vec3Int(1, 1, 1),
184+
path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/color/1"))
185+
),
186+
MagLocator(
187+
mag = Vec3Int(2, 2, 1),
188+
path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/color/2-2-1"))
189+
),
190+
MagLocator(
191+
mag = Vec3Int(4, 4, 1),
192+
path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/color/4-4-1"))
193+
),
194+
MagLocator(
195+
mag = Vec3Int(8, 8, 2),
196+
path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/color/8-8-2"))
197+
),
198+
MagLocator(
199+
mag = Vec3Int(16, 16, 4),
200+
path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/color/16-16-4"))
201+
)
202+
)
203+
),
204+
StaticSegmentationLayer(
205+
name = "segmentation",
206+
dataFormat = DataFormat.zarr3,
207+
boundingBox = BoundingBox(Vec3Int(3072, 3072, 512), 1024, 1024, 1024),
208+
elementClass = ElementClass.uint32,
209+
mags = List(
210+
MagLocator(
211+
mag = Vec3Int(1, 1, 1),
212+
path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/segmentation/1"))
213+
),
214+
MagLocator(
215+
mag = Vec3Int(2, 2, 1),
216+
path =
217+
Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/segmentation/2-2-1"))
218+
),
219+
MagLocator(
220+
mag = Vec3Int(4, 4, 1),
221+
path =
222+
Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/segmentation/4-4-1"))
223+
),
224+
MagLocator(
225+
mag = Vec3Int(8, 8, 2),
226+
path =
227+
Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/segmentation/8-8-2"))
228+
),
229+
MagLocator(
230+
mag = Vec3Int(16, 16, 4),
231+
path =
232+
Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/segmentation/16-16-4"))
233+
)
234+
),
235+
largestSegmentId = Some(2504697),
236+
)
237+
),
238+
scale = VoxelSize(Vec3Double(11.239999771118164, 11.239999771118164, 28), LengthUnit.nanometer)
239+
)
240+
241+
private val defaultDataset = Dataset(
242+
_id = ObjectId("68b80290d4000090f8f4aa62"),
243+
_dataStore = defaultDataStore.name,
244+
_organization = defaultOrganization._id,
245+
_publication = Some(defaultPublication._id),
246+
_uploader = Some(defaultUser._id),
247+
_folder = defaultOrganization._rootFolder,
248+
inboxSourceHash = Some(defaultDataSource.hashCode()),
249+
defaultViewConfiguration = None,
250+
adminViewConfiguration = None,
251+
description = None,
252+
directoryName = defaultDataSource.id.directoryName,
253+
isPublic = false,
254+
isUsable = true,
255+
isVirtual = true,
256+
name = "l4_sample_remote",
257+
voxelSize = Some(defaultDataSource.scale),
258+
sharingToken = None,
259+
status = "",
260+
logoUrl = None,
261+
)
159262

160263
def insert: Fox[Unit] =
161264
for {
@@ -175,6 +278,7 @@ Samplecountry
175278
_ <- insertTaskType()
176279
_ <- insertProject()
177280
_ <- insertPublication()
281+
_ <- insertDataset()
178282
_ <- insertAiModel()
179283
} yield ()
180284

@@ -276,6 +380,15 @@ Samplecountry
276380
} else Fox.successful(())
277381
}
278382

383+
private def insertDataset(): Fox[Unit] = datasetDAO.findOne(defaultDataset._id).shiftBox.flatMap { maybeDataset =>
384+
if (maybeDataset.isEmpty) {
385+
for {
386+
_ <- datasetDAO.insertOne(defaultDataset)
387+
_ <- datasetLayerDAO.updateLayers(defaultDataset._id, defaultDataSource)
388+
} yield ()
389+
} else Fox.successful(())
390+
}
391+
279392
private def insertAiModel(): Fox[Unit] = aiModelDAO.findAll.flatMap { aiModels =>
280393
if (aiModels.isEmpty) {
281394
aiModelDAO.insertOne(defaultAiModel)

‎app/models/dataset/DatasetService.scala‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ import play.api.http.Status.NOT_FOUND
2727
import play.api.i18n.{Messages, MessagesProvider}
2828
import play.api.libs.json.{JsObject, Json}
2929
import security.RandomIDGenerator
30-
import utils.WkConf
3130

3231
import javax.inject.Inject
3332
import scala.concurrent.duration._
@@ -45,8 +44,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
4544
teamService: TeamService,
4645
thumbnailCachingService: ThumbnailCachingService,
4746
userService: UserService,
48-
rpc: RPC,
49-
conf: WkConf)(implicit ec: ExecutionContext)
47+
rpc: RPC)(implicit ec: ExecutionContext)
5048
extends FoxImplicits
5149
with LazyLogging {
5250

@@ -216,7 +214,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
216214
case Some(foundDataset) => // This only returns None for Datasets that are present on a normal Datastore but also got reported from a scratch Datastore
217215
updateDataSourceDifferentDataStore(foundDataset, dataSource, dataStore)
218216
case _ =>
219-
insertNewDataset(dataSource, dataSource.id.directoryName, dataStore).map(Some(_))
217+
createDataset(dataStore, ObjectId.generate, dataSource.id.directoryName, dataSource).map(ds => Some(ds._id))
220218
}
221219
}
222220
}
@@ -262,11 +260,6 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
262260
}
263261
}).flatten
264262

265-
private def insertNewDataset(dataSource: DataSource, datasetName: String, dataStore: DataStore) =
266-
publicationForFirstDataset.flatMap { publicationId: Option[ObjectId] =>
267-
createDataset(dataStore, ObjectId.generate, datasetName, dataSource, publicationId).map(_._id)
268-
}
269-
270263
def updateDataSourceFromUserChanges(dataset: Dataset, dataSourceUpdates: UsableDataSource)(
271264
implicit ctx: DBAccessContext,
272265
mp: MessagesProvider): Fox[Unit] =
@@ -368,16 +361,6 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
368361
}
369362
}
370363

371-
private def publicationForFirstDataset: Fox[Option[ObjectId]] =
372-
if (conf.WebKnossos.SampleOrganization.enabled) {
373-
datasetDAO.isEmpty.map { isEmpty =>
374-
if (isEmpty)
375-
Some(ObjectId("5c766bec6c01006c018c7459"))
376-
else
377-
None
378-
}
379-
} else Fox.successful(None)
380-
381364
def deactivateUnreportedDataSources(reportedDatasetIds: List[ObjectId],
382365
dataStore: DataStore,
383366
organizationId: Option[String]): Fox[Unit] =

‎unreleased_changes/8905.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
### Added
2+
- Improved speed of ad-hoc mesh computation by 10–20%

‎util/src/main/scala/com/scalableminds/util/geometry/Vec3Float.scala‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,12 @@ case class Vec3Float(x: Float, y: Float, z: Float) {
2525
}
2626

2727
object Vec3Float {
28+
def apply(p: Vec3Int): Vec3Float =
29+
Vec3Float(p.x.toFloat, p.y.toFloat, p.z.toFloat)
30+
31+
def apply(p: Vec3Double): Vec3Float =
32+
Vec3Float(p.x.toFloat, p.y.toFloat, p.z.toFloat)
33+
2834
implicit object Vec3FloatReads extends Reads[Vec3Float] {
2935
def reads(json: JsValue): JsResult[Vec3Float] = json match {
3036
case JsArray(ts) if ts.size == 3 =>

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Codecs.scala‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,15 @@ class BloscCodec(cname: String, clevel: Int, shuffle: CompressionSetting, typesi
130130
override def decode(bytes: Array[Byte]): Array[Byte] = compressor.decompress(bytes)
131131
}
132132

133+
object BloscCodec {
134+
def fromConfiguration(configuration: BloscCodecConfiguration): BloscCodec =
135+
new BloscCodec(configuration.cname,
136+
configuration.clevel,
137+
configuration.shuffle,
138+
configuration.typesize,
139+
configuration.blocksize)
140+
}
141+
133142
class GzipCodec(level: Int) extends BytesToBytesCodec {
134143

135144
// https://zarr-specs.readthedocs.io/en/latest/v3/codecs/gzip/v1.0.html
@@ -236,12 +245,21 @@ object BloscCodecConfiguration {
236245
implicit val jsonFormat: OFormat[BloscCodecConfiguration] = Json.format[BloscCodecConfiguration]
237246
val name = "blosc"
238247

239-
def shuffleSettingFromInt(shuffle: Int): String = shuffle match {
248+
private def shuffleSettingFromInt(shuffle: Int): String = shuffle match {
240249
case 0 => "noshuffle"
241250
case 1 => "shuffle"
242251
case 2 => "bitshuffle"
243252
case _ => ???
244253
}
254+
255+
lazy val defaultForWKZarrOutput: BloscCodecConfiguration =
256+
BloscCodecConfiguration(
257+
BloscCompressor.defaultCname.getValue,
258+
BloscCompressor.defaultCLevel,
259+
StringCompressionSetting(BloscCodecConfiguration.shuffleSettingFromInt(BloscCompressor.defaultShuffle.getValue)),
260+
Some(BloscCompressor.defaultTypesize),
261+
BloscCompressor.defaultBlocksize
262+
)
245263
}
246264

247265
final case class GzipCodecConfiguration(level: Int) extends CodecConfiguration {
@@ -319,13 +337,12 @@ object CodecTreeExplorer {
319337
def findOne(condition: Function[CodecConfiguration, Boolean])(
320338
codecs: Seq[CodecConfiguration]): Option[CodecConfiguration] = {
321339
val results: Seq[Option[CodecConfiguration]] = codecs.map {
322-
case s: ShardingCodecConfiguration => {
340+
case s: ShardingCodecConfiguration =>
323341
if (condition(s)) {
324342
Some(s)
325343
} else {
326344
findOne(condition)(s.codecs)
327345
}
328-
}
329346
case c: CodecConfiguration => Some(c).filter(condition)
330347
}
331348
results.flatten.headOption

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3Array.scala‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,12 @@ class Zarr3Array(vaultPath: VaultPath,
6969

7070
private def initializeCodecs(codecSpecs: Seq[CodecConfiguration]): (Option[ShardingCodec], Seq[Codec], Seq[Codec]) = {
7171
val outerCodecs = codecSpecs.map {
72-
case BytesCodecConfiguration(endian) => new BytesCodec(endian)
73-
case TransposeCodecConfiguration(order) => new TransposeCodec(order)
74-
case BloscCodecConfiguration(cname, clevel, shuffle, typesize, blocksize) =>
75-
new BloscCodec(cname, clevel, shuffle, typesize, blocksize)
76-
case GzipCodecConfiguration(level) => new GzipCodec(level)
77-
case ZstdCodecConfiguration(level, checksum) => new ZstdCodec(level, checksum)
78-
case Crc32CCodecConfiguration => new Crc32CCodec
72+
case BytesCodecConfiguration(endian) => new BytesCodec(endian)
73+
case TransposeCodecConfiguration(order) => new TransposeCodec(order)
74+
case bloscConfiguration: BloscCodecConfiguration => BloscCodec.fromConfiguration(bloscConfiguration)
75+
case GzipCodecConfiguration(level) => new GzipCodec(level)
76+
case ZstdCodecConfiguration(level, checksum) => new ZstdCodec(level, checksum)
77+
case Crc32CCodecConfiguration => new Crc32CCodec
7978
case ShardingCodecConfiguration(chunk_shape, codecs, index_codecs, index_location) =>
8079
new ShardingCodec(chunk_shape, codecs, index_codecs, index_location)
8180
}

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mcubes/MarchingCubes.scala‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package com.scalableminds.webknossos.datastore.services.mcubes
22

3-
import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int}
3+
import com.scalableminds.util.geometry.{BoundingBox, Vec3Float, Vec3Int}
44

55
import scala.collection.mutable
66

@@ -10,9 +10,9 @@ object MarchingCubes {
1010
dataDimensions: Vec3Int,
1111
boundingBox: BoundingBox,
1212
segmentId: T,
13-
offset: Vec3Double,
14-
scale: Vec3Double,
15-
vertexBuffer: mutable.ArrayBuffer[Vec3Double]): Unit = {
13+
offset: Vec3Float,
14+
scale: Vec3Float,
15+
vertexBuffer: mutable.ArrayBuffer[Float]): Unit = {
1616

1717
def getVoxelData(x: Int, y: Int, z: Int): T =
1818
data(x + (dataDimensions.x * y) + (dataDimensions.x * dataDimensions.y * z))
@@ -49,9 +49,10 @@ object MarchingCubes {
4949
if (getVoxelData(x, y + 1, z + 1) == segmentId) cubeIndex |= 128
5050
if (getVoxelData(x + 1, y + 1, z + 1) == segmentId) cubeIndex |= 64
5151

52-
val position = Vec3Double(x, y, z)
5352
MarchingCubesTable.triangleTable(cubeIndex).foreach { edgeDelta =>
54-
vertexBuffer += (position + edgeDelta + offset) * scale
53+
vertexBuffer += (x + edgeDelta.x + offset.x) * scale.x
54+
vertexBuffer += (y + edgeDelta.y + offset.y) * scale.y
55+
vertexBuffer += (z + edgeDelta.z + offset.z) * scale.z
5556
}
5657
}
5758
}

0 commit comments

Comments
 (0)