Skip to content

Commit a410e8f

Browse files
frcrothfrcroth
authored
Store layer-specific attachments (#8598)
1 parent a9e120d commit a410e8f

File tree

17 files changed

+281
-18
lines changed

17 files changed

+281
-18
lines changed

‎MIGRATIONS.unreleased.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,5 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
99
[Commits](https://github.com/scalableminds/webknossos/compare/25.06.0...HEAD)
1010

1111
### Postgres Evolutions:
12+
13+
- [134-dataset-layer-attachments.sql](conf/evolutions/134-dataset-layer-attachments.sql)

‎app/models/dataset/Dataset.scala‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ import com.scalableminds.webknossos.datastore.models.datasource.{
1919
CoordinateTransformationType,
2020
DataSourceId,
2121
ElementClass,
22+
LayerAttachment,
23+
LayerAttachmentType,
2224
ThinPlateSplineCorrespondences,
2325
DataLayerLike => DataLayer
2426
}
@@ -841,11 +843,11 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte
841843

842844
}
843845

844-
class DatasetLayerDAO @Inject()(
845-
sqlClient: SqlClient,
846-
datasetMagsDAO: DatasetMagsDAO,
847-
datasetCoordinateTransformationsDAO: DatasetCoordinateTransformationsDAO,
848-
datasetLayerAdditionalAxesDAO: DatasetLayerAdditionalAxesDAO)(implicit ec: ExecutionContext)
846+
class DatasetLayerDAO @Inject()(sqlClient: SqlClient,
847+
datasetMagsDAO: DatasetMagsDAO,
848+
datasetCoordinateTransformationsDAO: DatasetCoordinateTransformationsDAO,
849+
datasetLayerAdditionalAxesDAO: DatasetLayerAdditionalAxesDAO,
850+
datasetLayerAttachmentsDAO: DatasetLayerAttachmentsDAO)(implicit ec: ExecutionContext)
849851
extends SimpleSQLDAO(sqlClient) {
850852

851853
private def parseRow(row: DatasetLayersRow, datasetId: ObjectId): Fox[DataLayer] = {
@@ -970,6 +972,7 @@ class DatasetLayerDAO @Inject()(
970972
_ <- datasetMagsDAO.updateMags(datasetId, source.toUsable.map(_.dataLayers))
971973
_ <- datasetCoordinateTransformationsDAO.updateCoordinateTransformations(datasetId,
972974
source.toUsable.map(_.dataLayers))
975+
_ <- datasetLayerAttachmentsDAO.updateAttachments(datasetId, source.toUsable.map(_.dataLayers))
973976
_ <- datasetLayerAdditionalAxesDAO.updateAdditionalAxes(datasetId, source.toUsable.map(_.dataLayers))
974977
} yield ()
975978
}
@@ -1010,6 +1013,37 @@ class DatasetLastUsedTimesDAO @Inject()(sqlClient: SqlClient)(implicit ec: Execu
10101013
}
10111014
}
10121015

1016+
class DatasetLayerAttachmentsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
1017+
extends SimpleSQLDAO(sqlClient) {
1018+
def updateAttachments(datasetId: ObjectId, dataLayersOpt: Option[List[DataLayer]]): Fox[Unit] = {
1019+
def insertQuery(attachment: LayerAttachment, layerName: String, fileType: String) =
1020+
q"""INSERT INTO webknossos.dataset_layer_attachments(_dataset, layerName, name, path, type, dataFormat)
1021+
VALUES($datasetId, $layerName, ${attachment.name}, ${attachment.path.toString}, $fileType::webknossos.LAYER_ATTACHMENT_TYPE,
1022+
${attachment.dataFormat}::webknossos.LAYER_ATTACHMENT_DATAFORMAT)""".asUpdate
1023+
val clearQuery =
1024+
q"DELETE FROM webknossos.dataset_layer_attachments WHERE _dataset = $datasetId".asUpdate
1025+
val insertQueries = dataLayersOpt.getOrElse(List.empty).flatMap { layer: DataLayer =>
1026+
layer.attachments match {
1027+
case Some(attachments) =>
1028+
attachments.agglomerates.map { agglomerate =>
1029+
insertQuery(agglomerate, layer.name, LayerAttachmentType.agglomerate.toString)
1030+
} ++ attachments.connectomes.map { connectome =>
1031+
insertQuery(connectome, layer.name, LayerAttachmentType.connectome.toString)
1032+
} ++ attachments.segmentIndex.map { segmentIndex =>
1033+
insertQuery(segmentIndex, layer.name, LayerAttachmentType.segmentIndex.toString)
1034+
} ++ attachments.meshes.map { mesh =>
1035+
insertQuery(mesh, layer.name, LayerAttachmentType.mesh.toString)
1036+
} ++ attachments.cumsum.map { cumsumFile =>
1037+
insertQuery(cumsumFile, layer.name, LayerAttachmentType.cumsum.toString)
1038+
}
1039+
case None =>
1040+
List.empty
1041+
}
1042+
}
1043+
replaceSequentiallyAsTransaction(clearQuery, insertQueries)
1044+
}
1045+
}
1046+
10131047
class DatasetCoordinateTransformationsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
10141048
extends SimpleSQLDAO(sqlClient) {
10151049
private def parseRow(row: DatasetLayerCoordinatetransformationsRow): Fox[CoordinateTransformation] =

‎app/models/dataset/DatasetService.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
3636
datasetLastUsedTimesDAO: DatasetLastUsedTimesDAO,
3737
datasetDataLayerDAO: DatasetLayerDAO,
3838
datasetMagsDAO: DatasetMagsDAO,
39+
datasetLayerAttachmentsDAO: DatasetLayerAttachmentsDAO,
3940
teamDAO: TeamDAO,
4041
folderDAO: FolderDAO,
4142
dataStoreService: DataStoreService,

‎conf/evolutions/134-dataset-layer-attachments.sql‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
START TRANSACTION;
2+
3+
do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 133, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
4+
5+
CREATE TYPE webknossos.LAYER_ATTACHMENT_TYPE AS ENUM ('agglomerate', 'connectome', 'segmentIndex', 'mesh', 'cumsum');
6+
CREATE TYPE webknossos.LAYER_ATTACHMENT_DATAFORMAT AS ENUM ('hdf5', 'zarr3', 'json');
7+
8+
CREATE TABLE webknossos.dataset_layer_attachments(
9+
_dataset TEXT CONSTRAINT _dataset_objectId CHECK (_dataset ~ '^[0-9a-f]{24}$') NOT NULL,
10+
layerName TEXT NOT NULL,
11+
name TEXT NOT NULL,
12+
path TEXT NOT NULL,
13+
type webknossos.LAYER_ATTACHMENT_TYPE NOT NULL,
14+
dataFormat webknossos.LAYER_ATTACHMENT_DATAFORMAT NOT NULL,
15+
PRIMARY KEY(_dataset, layerName, name, type)
16+
);
17+
18+
ALTER TABLE webknossos.dataset_layer_attachments
19+
ADD CONSTRAINT dataset_ref FOREIGN KEY(_dataset) REFERENCES webknossos.datasets(_id) DEFERRABLE,
20+
ADD CONSTRAINT layer_ref FOREIGN KEY(_dataset, layerName) REFERENCES webknossos.dataset_layers(_dataset, name) ON DELETE CASCADE DEFERRABLE;
21+
22+
UPDATE webknossos.releaseInformation SET schemaVersion = 134;
23+
24+
COMMIT TRANSACTION;

‎conf/evolutions/reversions/134-dataset-layer-attachments.sql‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
START TRANSACTION;
2+
3+
do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 134, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
4+
5+
DROP TABLE IF EXISTS webknossos.dataset_layer_attachments;
6+
DROP TYPE IF EXISTS webknossos.LAYER_ATTACHMENT_TYPE;
7+
DROP TYPE IF EXISTS webknossos.LAYER_ATTACHMENT_DATAFORMAT;
8+
9+
UPDATE webknossos.releaseInformation SET schemaVersion = 133;
10+
11+
COMMIT TRANSACTION;

‎tools/postgres/schema.sql‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ CREATE TABLE webknossos.releaseInformation (
2121
schemaVersion BIGINT NOT NULL
2222
);
2323

24-
INSERT INTO webknossos.releaseInformation(schemaVersion) values(133);
24+
INSERT INTO webknossos.releaseInformation(schemaVersion) values(134);
2525
COMMIT TRANSACTION;
2626

2727

@@ -162,6 +162,18 @@ CREATE TABLE webknossos.dataset_layer_additionalAxes(
162162
index INT NOT NULL
163163
);
164164

165+
CREATE TYPE webknossos.LAYER_ATTACHMENT_TYPE AS ENUM ('agglomerate', 'connectome', 'segmentIndex', 'mesh', 'cumsum');
166+
CREATE TYPE webknossos.LAYER_ATTACHMENT_DATAFORMAT AS ENUM ('hdf5', 'zarr3', 'json');
167+
CREATE TABLE webknossos.dataset_layer_attachments(
168+
_dataset TEXT CONSTRAINT _dataset_objectId CHECK (_dataset ~ '^[0-9a-f]{24}$') NOT NULL,
169+
layerName TEXT NOT NULL,
170+
name TEXT NOT NULL,
171+
path TEXT NOT NULL,
172+
type webknossos.LAYER_ATTACHMENT_TYPE NOT NULL,
173+
dataFormat webknossos.LAYER_ATTACHMENT_DATAFORMAT NOT NULL,
174+
PRIMARY KEY(_dataset, layerName, name, type)
175+
);
176+
165177
CREATE TABLE webknossos.dataset_allowedTeams(
166178
_dataset TEXT CONSTRAINT _dataset_objectId CHECK (_dataset ~ '^[0-9a-f]{24}$') NOT NULL,
167179
_team TEXT CONSTRAINT _team_objectId CHECK (_team ~ '^[0-9a-f]{24}$') NOT NULL,
@@ -202,6 +214,7 @@ CREATE TABLE webknossos.dataset_thumbnails(
202214
PRIMARY KEY (_dataset, dataLayerName, width, height, mappingName)
203215
);
204216

217+
205218
CREATE TYPE webknossos.DATASTORE_TYPE AS ENUM ('webknossos-store');
206219
CREATE TABLE webknossos.dataStores(
207220
name TEXT PRIMARY KEY NOT NULL CHECK (name ~* '^[A-Za-z0-9\-_\.]+$'),
@@ -885,6 +898,9 @@ ALTER TABLE webknossos.dataset_layer_coordinateTransformations
885898
ADD CONSTRAINT dataset_ref FOREIGN KEY(_dataset) REFERENCES webknossos.datasets(_id) DEFERRABLE;
886899
ALTER TABLE webknossos.dataset_layer_additionalAxes
887900
ADD CONSTRAINT dataset_ref FOREIGN KEY(_dataset) REFERENCES webknossos.datasets(_id) DEFERRABLE;
901+
ALTER TABLE webknossos.dataset_layer_attachments
902+
ADD CONSTRAINT dataset_ref FOREIGN KEY(_dataset) REFERENCES webknossos.datasets(_id) DEFERRABLE,
903+
ADD CONSTRAINT layer_ref FOREIGN KEY(_dataset, layerName) REFERENCES webknossos.dataset_layers(_dataset, name) ON DELETE CASCADE DEFERRABLE;
888904
ALTER TABLE webknossos.voxelytics_artifacts
889905
ADD FOREIGN KEY (_task) REFERENCES webknossos.voxelytics_tasks(_id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE;
890906
ALTER TABLE webknossos.voxelytics_runs

‎util/src/main/scala/com/scalableminds/util/mvc/Formatter.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ trait Formatter {
9292
if (includeStackTraces)
9393
s" Stack trace: ${TextUtils.stackTraceAsString(exception)} "
9494
else
95-
exception.toString
95+
s" ${exception.toString}"
9696
case _ => ""
9797
}
9898

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/N5DataLayers.scala‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ case class N5DataLayer(
3535
adminViewConfiguration: Option[LayerViewConfiguration] = None,
3636
coordinateTransformations: Option[List[CoordinateTransformation]] = None,
3737
override val numChannels: Option[Int] = Some(1),
38-
additionalAxes: Option[Seq[AdditionalAxis]] = None
38+
additionalAxes: Option[Seq[AdditionalAxis]] = None,
39+
attachments: Option[DatasetLayerAttachments] = None,
3940
) extends N5Layer
4041

4142
object N5DataLayer {
@@ -53,7 +54,8 @@ case class N5SegmentationLayer(
5354
adminViewConfiguration: Option[LayerViewConfiguration] = None,
5455
coordinateTransformations: Option[List[CoordinateTransformation]] = None,
5556
override val numChannels: Option[Int] = Some(1),
56-
additionalAxes: Option[Seq[AdditionalAxis]] = None
57+
additionalAxes: Option[Seq[AdditionalAxis]] = None,
58+
attachments: Option[DatasetLayerAttachments] = None,
5759
) extends SegmentationLayer
5860
with N5Layer
5961

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/PrecomputedDataLayers.scala‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ case class PrecomputedDataLayer(
3535
adminViewConfiguration: Option[LayerViewConfiguration] = None,
3636
coordinateTransformations: Option[List[CoordinateTransformation]] = None,
3737
override val numChannels: Option[Int] = Some(1),
38-
additionalAxes: Option[Seq[AdditionalAxis]] = None
38+
additionalAxes: Option[Seq[AdditionalAxis]] = None,
39+
attachments: Option[DatasetLayerAttachments] = None,
3940
) extends PrecomputedLayer
4041

4142
object PrecomputedDataLayer {
@@ -53,7 +54,8 @@ case class PrecomputedSegmentationLayer(
5354
adminViewConfiguration: Option[LayerViewConfiguration] = None,
5455
coordinateTransformations: Option[List[CoordinateTransformation]] = None,
5556
override val numChannels: Option[Int] = Some(1),
56-
additionalAxes: Option[Seq[AdditionalAxis]] = None
57+
additionalAxes: Option[Seq[AdditionalAxis]] = None,
58+
attachments: Option[DatasetLayerAttachments] = None,
5759
) extends SegmentationLayer
5860
with PrecomputedLayer
5961

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/WKWDataLayers.scala‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ case class WKWDataLayer(
4444
defaultViewConfiguration: Option[LayerViewConfiguration] = None,
4545
adminViewConfiguration: Option[LayerViewConfiguration] = None,
4646
coordinateTransformations: Option[List[CoordinateTransformation]] = None,
47-
additionalAxes: Option[Seq[AdditionalAxis]] = None
47+
additionalAxes: Option[Seq[AdditionalAxis]] = None,
48+
attachments: Option[DatasetLayerAttachments] = None,
4849
) extends WKWLayer
4950

5051
object WKWDataLayer {
@@ -61,7 +62,8 @@ case class WKWSegmentationLayer(
6162
defaultViewConfiguration: Option[LayerViewConfiguration] = None,
6263
adminViewConfiguration: Option[LayerViewConfiguration] = None,
6364
coordinateTransformations: Option[List[CoordinateTransformation]] = None,
64-
additionalAxes: Option[Seq[AdditionalAxis]] = None
65+
additionalAxes: Option[Seq[AdditionalAxis]] = None,
66+
attachments: Option[DatasetLayerAttachments] = None
6567
) extends SegmentationLayer
6668
with WKWLayer
6769

0 commit comments

Comments
 (0)