Make organization name its id, drop the mongo id (#7386)

* WIP: make organization name its id, drop the mongo id

* more usages

* Update LokiClient.scala

* fix compiler errors after merge

* prune scala usages of organizationName

* fix migration guide

* prune frontend usages of organizationName

* prune frontend usages of organization.name

* fix data type of orga references

* sql check, fix delete organization

* WIP: Add migration and revision

* format backend

* enable backend to parse old analytics schema for ingest events

* format backend

* update schema version

* finish first version of migration and revision

* fix schema & migration

- give explicit name to constraints to make the name more consistent and less implicit

* add migration for analyticsEvents' eventProperties

* add organizationName to LinkedLayerIdentifier to make it backwards compatible

* add legacy route to list datasets supporting the old get param organizationName when filtering for specific datasets & bump api version

* dont use direct get on box and throw jserror when no orgaid or organame is defined in analytics event

* format backend

* remove comments from evolution files & fix schema version

* fix migration, remove old now duplicate unique index

* readd unique contraint in reversion

* in reversion: migrate local join orga, open dataset & uplaod dataset back

* in reversion: fix fk constraints and views

* in reversion: fix user infos view

* migrate new code pulled in by merge to use orgaId and not name

* format backend

* update schema version & include warning in reversion

* rename organizationDisplayName to organizationName

* add changelog entry

* add migration entry

* remove auto generated message files for data and tracingstore

* update e2e db fixtures / initial db data

* rename left over orgDisplayName variable to orgName

* update snapshots to use new e2e your name last names

* fix typo and format code

* more relaxed analytics json reads

* drop default for id column

* add missing redirect to versioned.routes

* rename migration, undo changing worker job args

---------

Co-authored-by: Norman Rzepka <code@normanrz.com>
Co-authored-by: Michael Büßemeyer <michael.buessemeyer@student.hpi.de>
Co-authored-by: Michael Büßemeyer <MichaelBuessemeyer@users.noreply.github.com>

Author: 3 people

CHANGELOG.unreleased.md

@@ -42,6 +42,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - The warning about a mismatch between the scale of a pre-computed mesh and the dataset scale's factor now also considers all supported mags of the active segmentation layer. This reduces the false posive rate regarding this warning. [#7921](https://github.com/scalableminds/webknossos/pull/7921/)
 - It is no longer allowed to edit annotations of other organizations, even if they are set to public and to others-may-edit. [#7923](https://github.com/scalableminds/webknossos/pull/7923)
 - When proofreading segmentations, the user can now interact with super-voxels directly in the data viewports. Additionally, proofreading is significantly faster because the segmentation data doesn't have to be re-downloaded after each merge/split operation. [#7654](https://github.com/scalableminds/webknossos/pull/7654)
+- Changed internal data model changing an organization's name to id and its displayName to name. The previously existing id was removed. [#7386](https://github.com/scalableminds/webknossos/pull/7386)
 - Because of the way our models are trained, AI analysis and training is disabled for 2D and ND datasets, as well as for color layers with data type uInt24. [#7957](https://github.com/scalableminds/webknossos/pull/7957)
 - The overall performance was improved (especially for the segments tab). [#7958](https://github.com/scalableminds/webknossos/pull/7958)
 - The performance for the skeleton tab was improved. [#7989](https://github.com/scalableminds/webknossos/pull/7989)

MIGRATIONS.unreleased.md

@@ -12,3 +12,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 
 ### Postgres Evolutions:
 - [119-add-metadata-to-folders-and-datasets.sql](conf/evolutions/119-add-metadata-to-folders-and-datasets.sql)
+- [120-remove-old-organization-id.sql](conf/evolutions/120-remove-old-organization-id.sql)

app/controllers/AiModelController.scala

@@ -133,7 +133,7 @@ class AiModelController @Inject()(
         jobCommand = JobCommand.train_model
         commandArgs = Json.obj(
           "training_annotations" -> Json.toJson(trainingAnnotations),
-          "organization_name" -> organization.name,
+          "organization_name" -> organization._id,
           "model_id" -> modelId,
           "custom_workflow_provided_by_user" -> request.body.workflowYaml
         )
@@ -171,7 +171,7 @@ class AiModelController @Inject()(
         jobCommand = JobCommand.infer_with_model
         boundingBox <- BoundingBox.fromLiteral(request.body.boundingBox).toFox
         commandArgs = Json.obj(
-          "organization_name" -> organization.name,
+          "organization_name" -> organization._id,
           "dataset_name" -> dataset.name,
           "color_layer_name" -> request.body.colorLayerName,
           "bounding_box" -> boundingBox.toLiteral,

app/controllers/AnnotationController.scala

@@ -204,7 +204,7 @@ class AnnotationController @Inject()(
         newLayerName = request.body.name.getOrElse(AnnotationLayer.defaultNameForType(request.body.typ))
         _ <- bool2Fox(!annotation.annotationLayers.exists(_.name == newLayerName)) ?~> "annotation.addLayer.nameInUse"
         organization <- organizationDAO.findOne(request.identity._organization)
-        _ <- annotationService.addAnnotationLayer(annotation, organization.name, request.body)
+        _ <- annotationService.addAnnotationLayer(annotation, organization._id, request.body)
         updated <- provider.provideAnnotation(typ, id, request.identity)
         json <- annotationService.publicWrites(updated, Some(request.identity)) ?~> "annotation.write.failed"
       } yield JsonOk(json)
@@ -242,12 +242,12 @@ class AnnotationController @Inject()(
       } yield result
     }
 
-  def createExplorational(organizationName: String, datasetName: String): Action[List[AnnotationLayerParameters]] =
+  def createExplorational(organizationId: String, datasetName: String): Action[List[AnnotationLayerParameters]] =
     sil.SecuredAction.async(validateJson[List[AnnotationLayerParameters]]) { implicit request =>
       for {
-        organization <- organizationDAO.findOneByName(organizationName)(GlobalAccessContext) ?~> Messages(
+        organization <- organizationDAO.findOne(organizationId)(GlobalAccessContext) ?~> Messages(
           "organization.notFound",
-          organizationName) ~> NOT_FOUND
+          organizationId) ~> NOT_FOUND
         dataset <- datasetDAO.findOneByNameAndOrganization(datasetName, organization._id) ?~> Messages(
           "dataset.notFound",
           datasetName) ~> NOT_FOUND
@@ -262,16 +262,16 @@ class AnnotationController @Inject()(
       } yield JsonOk(json)
     }
 
-  def getSandbox(organizationName: String,
+  def getSandbox(organization: String,
                  datasetName: String,
                  typ: String,
                  sharingToken: Option[String]): Action[AnyContent] =
     sil.UserAwareAction.async { implicit request =>
       val ctx = URLSharing.fallbackTokenAccessContext(sharingToken) // users with dataset sharing token may also get a sandbox annotation
       for {
-        organization <- organizationDAO.findOneByName(organizationName)(GlobalAccessContext) ?~> Messages(
+        organization <- organizationDAO.findOne(organization)(GlobalAccessContext) ?~> Messages(
           "organization.notFound",
-          organizationName) ~> NOT_FOUND
+          organization) ~> NOT_FOUND
         dataset <- datasetDAO.findOneByNameAndOrganization(datasetName, organization._id)(ctx) ?~> Messages(
           "dataset.notFound",
           datasetName) ~> NOT_FOUND
@@ -301,7 +301,7 @@ class AnnotationController @Inject()(
         _ <- restrictions.allowUpdate(request.identity) ?~> "notAllowed" ~> FORBIDDEN
         annotation <- provider.provideAnnotation(typ, id, request.identity)
         organization <- organizationDAO.findOne(request.identity._organization)
-        _ <- annotationService.makeAnnotationHybrid(annotation, organization.name, fallbackLayerName) ?~> "annotation.makeHybrid.failed"
+        _ <- annotationService.makeAnnotationHybrid(annotation, organization._id, fallbackLayerName) ?~> "annotation.makeHybrid.failed"
         updated <- provider.provideAnnotation(typ, id, request.identity)
         json <- annotationService.publicWrites(updated, Some(request.identity)) ?~> "annotation.write.failed"
       } yield JsonOk(json)

app/controllers/AnnotationIOController.scala

@@ -103,14 +103,14 @@ class AnnotationIOController @Inject()(
           request.body.dataParts("createGroupForEachFile").headOption.contains("true")
         val overwritingDatasetName: Option[String] =
           request.body.dataParts.get("datasetName").flatMap(_.headOption)
-        val overwritingOrganizationName: Option[String] =
-          request.body.dataParts.get("organizationName").flatMap(_.headOption)
+        val overwritingOrganizationId: Option[String] =
+          request.body.dataParts.get("organizationId").flatMap(_.headOption)
         val attachedFiles = request.body.files.map(f => (f.ref.path.toFile, f.filename))
         val parsedFiles =
           annotationUploadService.extractFromFiles(attachedFiles,
                                                    useZipName = true,
                                                    overwritingDatasetName,
-                                                   overwritingOrganizationName)
+                                                   overwritingOrganizationId)
         val parsedFilesWrapped =
           annotationUploadService.wrapOrPrefixGroups(parsedFiles.parseResults, shouldCreateGroupForEachFile)
         val parseResultsFiltered: List[NmlParseResult] = parsedFilesWrapped.filter(_.succeeded)
@@ -225,15 +225,15 @@ class AnnotationIOController @Inject()(
       wkUrl: String)(implicit mp: MessagesProvider, ctx: DBAccessContext): Fox[Dataset] =
     for {
       datasetName <- assertAllOnSameDataset(skeletonTracings, volumeTracings) ?~> "nml.file.differentDatasets"
-      organizationNameOpt <- assertAllOnSameOrganization(skeletonTracings, volumeTracings) ?~> "nml.file.differentDatasets"
-      organizationIdOpt <- Fox.runOptional(organizationNameOpt) {
-        organizationDAO.findOneByName(_)(GlobalAccessContext).map(_._id)
+      organizationIdOpt <- assertAllOnSameOrganization(skeletonTracings, volumeTracings) ?~> "nml.file.differentDatasets"
+      organizationIdOpt <- Fox.runOptional(organizationIdOpt) {
+        organizationDAO.findOne(_)(GlobalAccessContext).map(_._id)
       } ?~> (if (wkUrl.nonEmpty && conf.Http.uri != wkUrl) {
-               Messages("organization.notFound.wrongHost", organizationNameOpt.getOrElse(""), wkUrl, conf.Http.uri)
-             } else { Messages("organization.notFound", organizationNameOpt.getOrElse("")) }) ~>
+               Messages("organization.notFound.wrongHost", organizationIdOpt.getOrElse(""), wkUrl, conf.Http.uri)
+             } else { Messages("organization.notFound", organizationIdOpt.getOrElse("")) }) ~>
         NOT_FOUND
       organizationId <- Fox.fillOption(organizationIdOpt) {
-        datasetDAO.getOrganizationForDataset(datasetName)(GlobalAccessContext)
+        datasetDAO.getOrganizationIdForDataset(datasetName)(GlobalAccessContext)
       } ?~> Messages("dataset.noAccess", datasetName) ~> FORBIDDEN
       dataset <- datasetDAO.findOneByNameAndOrganization(datasetName, organizationId) ?~> (if (wkUrl.nonEmpty && conf.Http.uri != wkUrl) {
                                                                                              Messages(
@@ -277,11 +277,11 @@ class AnnotationIOController @Inject()(
 
   private def assertAllOnSameOrganization(skeletons: List[SkeletonTracing],
                                           volumes: List[VolumeTracing]): Fox[Option[String]] = {
-    // Note that organizationNames are optional. Tracings with no organization attribute are ignored here
-    val organizationNames = skeletons.flatMap(_.organizationName) ::: volumes.flatMap(_.organizationName)
+    // Note that organizationIds are optional. Tracings with no organization attribute are ignored here
+    val organizationIds = skeletons.flatMap(_.organizationId) ::: volumes.flatMap(_.organizationId)
     for {
-      _ <- Fox.runOptional(organizationNames.headOption)(name => bool2Fox(organizationNames.forall(_ == name)))
-    } yield organizationNames.headOption
+      _ <- Fox.runOptional(organizationIds.headOption)(name => bool2Fox(organizationIds.forall(_ == name)))
+    } yield organizationIds.headOption
   }
 
   private def adaptVolumeTracingsToFallbackLayer(volumeLayersGrouped: List[List[UploadedVolumeLayer]],
@@ -297,7 +297,7 @@ class AnnotationIOController @Inject()(
             adaptedTracing <- adaptPropertiesToFallbackLayer(volumeLayer.tracing,
                                                              dataSource,
                                                              dataset,
-                                                             organization.name,
+                                                             organization._id,
                                                              remoteDataStoreClient)
             adaptedAnnotationLayer = volumeLayer.copy(tracing = adaptedTracing)
           } yield adaptedAnnotationLayer
@@ -309,7 +309,7 @@ class AnnotationIOController @Inject()(
       volumeTracing: VolumeTracing,
       dataSource: GenericDataSource[T],
       dataset: Dataset,
-      organizationName: String,
+      organizationId: String,
       remoteDataStoreClient: WKRemoteDataStoreClient): Fox[VolumeTracing] = {
     val fallbackLayerOpt = dataSource.dataLayers.flatMap {
       case layer: SegmentationLayer if volumeTracing.fallbackLayer contains layer.name         => Some(layer)
@@ -323,7 +323,7 @@ class AnnotationIOController @Inject()(
       .map(layer => elementClassToProto(layer.elementClass))
       .getOrElse(elementClassToProto(VolumeTracingDefaults.elementClass))
     for {
-      tracingCanHaveSegmentIndex <- canHaveSegmentIndex(organizationName,
+      tracingCanHaveSegmentIndex <- canHaveSegmentIndex(organizationId,
                                                         dataset.name,
                                                         fallbackLayerOpt.map(_.name),
                                                         remoteDataStoreClient)
@@ -341,13 +341,13 @@ class AnnotationIOController @Inject()(
   }
 
   private def canHaveSegmentIndex(
-      organizationName: String,
+      organizationId: String,
       datasetName: String,
       fallbackLayerName: Option[String],
       remoteDataStoreClient: WKRemoteDataStoreClient)(implicit ec: ExecutionContext): Fox[Boolean] =
     fallbackLayerName match {
       case Some(layerName) =>
-        remoteDataStoreClient.hasSegmentIndexFile(organizationName, datasetName, layerName)
+        remoteDataStoreClient.hasSegmentIndexFile(organizationId, datasetName, layerName)
       case None =>
         Fox.successful(true)
     }
@@ -411,9 +411,7 @@ class AnnotationIOController @Inject()(
 
     // Note: volumeVersion cannot currently be supplied per layer, see https://github.com/scalableminds/webknossos/issues/5925
 
-    def skeletonToTemporaryFile(dataset: Dataset,
-                                annotation: Annotation,
-                                organizationName: String): Fox[TemporaryFile] =
+    def skeletonToTemporaryFile(dataset: Dataset, annotation: Annotation, organizationId: String): Fox[TemporaryFile] =
       for {
         tracingStoreClient <- tracingStoreService.clientFor(dataset)
         fetchedAnnotationLayers <- Fox.serialCombined(annotation.skeletonAnnotationLayers)(
@@ -426,7 +424,7 @@ class AnnotationIOController @Inject()(
           Some(annotation),
           dataset.voxelSize,
           None,
-          organizationName,
+          organizationId,
           conf.Http.uri,
           dataset.name,
           Some(user),
@@ -443,7 +441,7 @@ class AnnotationIOController @Inject()(
     def volumeOrHybridToTemporaryFile(dataset: Dataset,
                                       annotation: Annotation,
                                       name: String,
-                                      organizationName: String): Fox[TemporaryFile] =
+                                      organizationId: String): Fox[TemporaryFile] =
       for {
         tracingStoreClient <- tracingStoreService.clientFor(dataset)
         fetchedVolumeLayers: List[FetchedAnnotationLayer] <- Fox.serialCombined(annotation.volumeAnnotationLayers) {
@@ -466,7 +464,7 @@ class AnnotationIOController @Inject()(
           Some(annotation),
           dataset.voxelSize,
           None,
-          organizationName,
+          organizationId,
           conf.Http.uri,
           dataset.name,
           Some(user),
@@ -491,11 +489,11 @@ class AnnotationIOController @Inject()(
     def annotationToTemporaryFile(dataset: Dataset,
                                   annotation: Annotation,
                                   name: String,
-                                  organizationName: String): Fox[TemporaryFile] =
+                                  organizationId: String): Fox[TemporaryFile] =
       if (annotation.tracingType == TracingType.skeleton)
-        skeletonToTemporaryFile(dataset, annotation, organizationName) ?~> "annotation.download.skeletonToFile.failed"
+        skeletonToTemporaryFile(dataset, annotation, organizationId) ?~> "annotation.download.skeletonToFile.failed"
       else
-        volumeOrHybridToTemporaryFile(dataset, annotation, name, organizationName) ?~> "annotation.download.hybridToFile.failed"
+        volumeOrHybridToTemporaryFile(dataset, annotation, name, organizationId) ?~> "annotation.download.hybridToFile.failed"
 
     def exportExtensionForAnnotation(annotation: Annotation): String =
       if (annotation.tracingType == TracingType.skeleton)
@@ -519,7 +517,7 @@ class AnnotationIOController @Inject()(
       _ <- restrictions.allowDownload(issuingUser) ?~> "annotation.download.notAllowed" ~> FORBIDDEN
       dataset <- datasetDAO.findOne(annotation._dataset)(GlobalAccessContext) ?~> "dataset.notFoundForAnnotation" ~> NOT_FOUND
       organization <- organizationDAO.findOne(dataset._organization)(GlobalAccessContext) ?~> "organization.notFound" ~> NOT_FOUND
-      temporaryFile <- annotationToTemporaryFile(dataset, annotation, name, organization.name) ?~> "annotation.writeTemporaryFile.failed"
+      temporaryFile <- annotationToTemporaryFile(dataset, annotation, name, organization._id) ?~> "annotation.writeTemporaryFile.failed"
     } yield {
       Ok.sendFile(temporaryFile, inline = false)
         .as(mimeType)

app/controllers/Application.scala

@@ -64,8 +64,7 @@ class Application @Inject()(actorSystem: ActorSystem,
     for {
       organization <- organizationDAO.findOne(request.identity._organization)
       userEmail <- userService.emailFor(request.identity)
-      _ = Mailer ! Send(
-        defaultMails.helpMail(request.identity, userEmail, organization.displayName, message, currentUrl))
+      _ = Mailer ! Send(defaultMails.helpMail(request.identity, userEmail, organization.name, message, currentUrl))
     } yield Ok
   }