Restructure list annotations query to guide Postgres to a more efficient query plan

CHANGELOG.unreleased.md

@@ -27,6 +27,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Fixed that a warning message about a newer version of an annotation was shown multiple times. [#8486](https://github.com/scalableminds/webknossos/pull/8486)
 - Fixed a bug where segment statistics would sometimes be wrong in case of an on-disk segmentation fallback layer with segment index file. [#8460](https://github.com/scalableminds/webknossos/pull/8460)
 - Fixed a bug where sometimes outdated segment statistics would be displayed. [#8460](https://github.com/scalableminds/webknossos/pull/8460)
+- Fixed a bug where the annotation list would sometimes load very long if you have many annotations. [#8498](https://github.com/scalableminds/webknossos/pull/8498)
 
 ### Removed
 

app/models/annotation/Annotation.scala

@@ -399,62 +399,99 @@ class AnnotationDAO @Inject()(sqlClient: SqlClient, annotationLayerDAO: Annotati
       stateQuery = getStateQuery(isFinished)
       userQuery = forUser.map(u => q"a._user = $u").getOrElse(q"TRUE")
       typQuery = q"a.typ = ${AnnotationType.Explorational}"
-
-      query = q"""WITH
-                    -- teams_agg is extracted to avoid left-join fanout.
-                    -- Note that only one of the joins in it has 1:n, so they can happen together
-                  teams_agg AS (
-                    SELECT
-                    a._id AS _annotation,
-                    ARRAY_REMOVE(ARRAY_AGG(t._id), null) AS team_ids,
-                    ARRAY_REMOVE(ARRAY_AGG(t.name), null) AS team_names,
-                    ARRAY_REMOVE(ARRAY_AGG(o._id), null) AS team_organization_ids
-                    FROM webknossos.annotations a
-                    LEFT JOIN webknossos.annotation_sharedteams ast ON ast._annotation = a._id
-                    LEFT JOIN webknossos.teams_ t ON ast._team = t._id
-                    LEFT JOIN webknossos.organizations_ o ON t._organization = o._id
-                    GROUP BY a._id
-                  )
-                  SELECT
-                    a._id,
-                    a.name,
-                    a.description,
-                    a._user,
-                    u.firstname,
-                    u.lastname,
-                    a.othersmayedit,
-                    teams_agg.team_ids,
-                    teams_agg.team_names,
-                    teams_agg.team_organization_ids,
-                    a.modified,
-                    a.tags,
-                    a.state,
-                    a.isLockedByOwner,
-                    d.name,
-                    a.typ,
-                    a.visibility,
-                    a.tracingtime,
-                    o._id,
-                    ARRAY_REMOVE(ARRAY_AGG(al.tracingid), null) AS tracing_ids,
-                    ARRAY_REMOVE(ARRAY_AGG(al.name), null) AS tracing_names,
-                    ARRAY_REMOVE(ARRAY_AGG(al.typ :: varchar), null) AS tracing_typs,
-                    ARRAY_REMOVE(ARRAY_AGG(al.statistics), null) AS annotation_layer_statistics
-                  FROM webknossos.annotations_ AS a
-                  JOIN webknossos.users_ u ON u._id = a._user
-                  JOIN teams_agg ON teams_agg._annotation = a._id
-                  JOIN webknossos.datasets_ d ON d._id = a._dataset
-                  JOIN webknossos.organizations_ AS o ON o._id = d._organization
-                  JOIN webknossos.annotation_layers AS al ON al._annotation = a._id
-                  WHERE $stateQuery AND $accessQuery AND $userQuery AND $typQuery
-                  GROUP BY
-                    a._id, a.name, a.description, a._user, a.othersmayedit, a.modified,
-                    a.tags, a.state,  a.islockedbyowner, a.typ, a.visibility, a.tracingtime,
-                    u.firstname, u.lastname,
-                    teams_agg.team_ids, teams_agg.team_names, teams_agg.team_organization_ids,
-                    d.name, o._id
-                  ORDER BY a._id DESC
-                  LIMIT $limit
-                  OFFSET ${pageNumber * limit}"""
+      query = q"""
+          -- We need to separate the querying of the annotation with all its inner joins from the 1:n join to collect the shared teams
+          -- This is to prevent left-join fanout.
+          -- Note that only one of the left joins in it has 1:n, so they can happen together
+          -- The WITH is structured this way round to get in the LIMIT early and not fetch the shared teams of all annotations first.
+          WITH an AS( -- select annotations with the relevant properties first
+            SELECT
+              a._id,
+              a.name,
+              a.description,
+              a._user,
+              u.firstname,
+              u.lastname,
+              a.othersmayedit,
+              a.modified,
+              a.tags,
+              a.state,
+              a.isLockedByOwner,
+              d.name AS datasetName,
+              a.typ,
+              a.visibility,
+              a.tracingtime,
+              o._id AS organizationId,
+              ARRAY_REMOVE(ARRAY_AGG(al.tracingid), null) AS tracing_ids,
+              ARRAY_REMOVE(ARRAY_AGG(al.name), null) AS tracing_names,
+              ARRAY_REMOVE(ARRAY_AGG(al.typ :: varchar), null) AS tracing_typs,
+              ARRAY_REMOVE(ARRAY_AGG(al.statistics), null) AS annotation_layer_statistics
+            FROM webknossos.annotations_ AS a
+            JOIN webknossos.users_ u ON u._id = a._user
+            JOIN webknossos.datasets_ d ON d._id = a._dataset
+            JOIN webknossos.organizations_ AS o ON o._id = d._organization
+            JOIN webknossos.annotation_layers AS al ON al._annotation = a._id
+            WHERE $stateQuery AND $accessQuery AND $userQuery AND $typQuery
+            GROUP BY
+              a._id, a.name, a.description, a._user, a.othersmayedit, a.modified,
+              a.tags, a.state,  a.islockedbyowner, a.typ, a.visibility, a.tracingtime,
+              u.firstname, u.lastname,
+              d.name, o._id
+            ORDER BY a._id DESC
+            LIMIT $limit
+            OFFSET ${pageNumber * limit}
+          )
+          SELECT  -- select now add the shared teams, and propagate everything to the output
+            an._id,
+            an.name,
+            an.description,
+            an._user,
+            an.firstname,
+            an.lastname,
+            an.othersmayedit,
+            ARRAY_REMOVE(ARRAY_AGG(t._id), null) AS team_ids,
+            ARRAY_REMOVE(ARRAY_AGG(t.name), null) AS team_names,
+            ARRAY_REMOVE(ARRAY_AGG(o._id), null) AS team_organization_ids,
+            an.modified,
+            an.tags,
+            an.state,
+            an.isLockedByOwner,
+            an.datasetName,
+            an.typ,
+            an.visibility,
+            an.tracingtime,
+            an.organizationId,
+            an.tracing_ids,
+            an.tracing_names,
+            an.tracing_typs,
+            an.annotation_layer_statistics
+          FROM an
+          LEFT JOIN webknossos.annotation_sharedteams ast ON ast._annotation = an._id
+          LEFT JOIN webknossos.teams_ t ON ast._team = t._id
+          LEFT JOIN webknossos.organizations_ o ON t._organization = o._id
+          GROUP BY
+            an._id,
+            an.name,
+            an.description,
+            an._user,
+            an.firstname,
+            an.lastname,
+            an.othersmayedit,
+            an.modified,
+            an.tags,
+            an.state,
+            an.isLockedByOwner,
+            an.datasetName,
+            an.typ,
+            an.visibility,
+            an.tracingtime,
+            an.organizationId,
+            an.tracing_ids,
+            an.tracing_names,
+            an.tracing_typs,
+            an.annotation_layer_statistics
+          ORDER BY an._id DESC
+         """
       rows <- run(query.as[AnnotationCompactInfo])
     } yield rows.toList