Merge branch 'main' into feature/2312-external-recommender-v2

* main: (94 commits)
  #4240 - Facilitate using DIAM serializer in isolation
  #4240 - Facilitate using DIAM serializer in isolation
  #4240 - Facilitate using DIAM serializer in isolation
  [maven-release-plugin] prepare for next development iteration
  [maven-release-plugin] prepare release inception-29.5
  #4230 - Upgrade dependencies
  #4230 - Upgrade dependencies
  #4235 - Direct access-by-URL to sidebar curation mode sometimes does not work
  #4229 - Better project template selection
  #4229 - Better project template selection
  #4232 - Drop evaluation page
  #4229 - Better project template selection
  #4230 - Upgrade dependencies
  #4229 - Better project template selection
  No issue: Update system requirements and improve pointers to Docker-based deployment
  No issue: Call out to admins to look at the admin guide when reaching the installation section in the users guide.
  #4215 - Ability to specify GID and UID for the user used inside the Docker image
  [maven-release-plugin] prepare for next development iteration
  [maven-release-plugin] prepare release inception-29.4
  #4226 - Display logout button on annotation page if dashboard access is disabled
  ...

% Conflicts:
%	inception/inception-imls-external/pom.xml

Jenkinsfile

@@ -1,8 +1,8 @@
 config = [
     agentLabel: '',
     maven: 'Maven 3',
-    jdk: 'Zulu 11',
-    extraMavenArguments: '',
+    jdk: 'Zulu 17',
+    extraMavenArguments: '-U -Ddkpro.core.testCachePath="${WORKSPACE}/cache/dkpro-core-datasets" -Dmaven.artifact.threads=15  -T 4',
     wipeWorkspaceBeforeBuild: true,
     wipeWorkspaceAfterBuild: true
   ]
@@ -28,7 +28,7 @@ pipeline {
   }
 
   agent {
-    label agentLabel
+    label params.agentLabel
   }
 
   tools {
@@ -97,7 +97,7 @@ pipeline {
             script {
               def mavenCommand = 'mvn ' +
                   params.extraMavenArguments +
-                  ' -B -Dmaven.test.failure.ignore=true clean verify';
+                  ' -B -Dmaven.test.failure.ignore=true -T 4 -Pjacoco clean verify javadoc:javadoc';
 
               if (isUnix()) {
                 sh script: mavenCommand
@@ -130,7 +130,7 @@ pipeline {
             script {
               def mavenCommand = 'mvn ' +
                 params.extraMavenArguments +
-                ' -B -Dmaven.test.failure.ignore=true clean verify'
+                ' -B -Dmaven.test.failure.ignore=true -Pjacoco,full-tests clean verify javadoc:javadoc'
 
               if (isUnix()) {
                 sh script: mavenCommand
@@ -155,7 +155,7 @@ pipeline {
   }
 
   post {
-    success {
+    always {
       script {
         if (params.wipeWorkspaceAfterBuild) {
           echo "Wiping workspace..."

inception/inception-active-learning/pom.xml

@@ -20,12 +20,16 @@
   <parent>
     <groupId>de.tudarmstadt.ukp.inception.app</groupId>
     <artifactId>inception-app</artifactId>
-    <version>29.0-SNAPSHOT</version>
+    <version>30.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
   <artifactId>inception-active-learning</artifactId>
   <name>INCEpTION - Recommendation - Active Learning</name>
   <dependencies>
+    <dependency>
+      <groupId>de.tudarmstadt.ukp.inception.app</groupId>
+      <artifactId>inception-annotation-storage-api</artifactId>
+    </dependency>
     <dependency>
       <groupId>de.tudarmstadt.ukp.inception.app</groupId>
       <artifactId>inception-ui-annotation</artifactId>
@@ -44,7 +48,7 @@
     </dependency>
     <dependency>
       <groupId>de.tudarmstadt.ukp.inception.app</groupId>
-      <artifactId>inception-api</artifactId>
+      <artifactId>inception-documents-api</artifactId>
     </dependency>
     <dependency>
       <groupId>de.tudarmstadt.ukp.inception.app</groupId>
@@ -54,6 +58,10 @@
       <groupId>de.tudarmstadt.ukp.inception.app</groupId>
       <artifactId>inception-model</artifactId>
     </dependency>
+    <dependency>
+      <groupId>de.tudarmstadt.ukp.inception.app</groupId>
+      <artifactId>inception-model-vdoc</artifactId>
+    </dependency>
     <dependency>
       <groupId>de.tudarmstadt.ukp.inception.app</groupId>
       <artifactId>inception-support</artifactId>

inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/ActiveLearningService.java

@@ -25,7 +25,6 @@
 import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument;
 import de.tudarmstadt.ukp.clarin.webanno.security.model.User;
 import de.tudarmstadt.ukp.inception.active.learning.ActiveLearningServiceImpl.ActiveLearningUserState;
-import de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecord;
 import de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordType;
 import de.tudarmstadt.ukp.inception.recommendation.api.model.SpanSuggestion;
 import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionGroup;
@@ -44,15 +43,6 @@ public interface ActiveLearningService
      */
     List<SuggestionGroup<SpanSuggestion>> getSuggestions(User aDataOwner, AnnotationLayer aLayer);
 
-    /**
-     * @param aRecord
-     *            record to check
-     * @return if the suggestions from which the given record was created (or an equivalent one) is
-     *         visible to the user. This is useful to check if the suggestion can be highlighted
-     *         when clicking on a history record.
-     */
-    boolean isSuggestionVisible(LearningRecord aRecord);
-
     /**
      * @return if the are any records of type {@link LearningRecordType#SKIPPED} in the history of
      *         the given layer for the given user.
@@ -64,8 +54,8 @@ public interface ActiveLearningService
      */
     boolean hasSkippedSuggestions(String aSessionOwner, User aDataOwner, AnnotationLayer aLayer);
 
-    void hideRejectedOrSkippedAnnotations(String aSessionOwner, User aDataOwner, AnnotationLayer aLayer,
-            boolean aFilterSkippedRecommendation,
+    void hideRejectedOrSkippedAnnotations(String aSessionOwner, User aDataOwner,
+            AnnotationLayer aLayer, boolean aFilterSkippedRecommendation,
             List<SuggestionGroup<SpanSuggestion>> aSuggestionGroups);
 
     Optional<Delta<SpanSuggestion>> generateNextSuggestion(String aSessionOwner, User aDataOwner,

inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/ActiveLearningServiceImpl.java

@@ -27,8 +27,8 @@
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.lang.invoke.MethodHandles;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 
 import org.slf4j.Logger;
@@ -37,7 +37,6 @@
 import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.transaction.annotation.Transactional;
 
-import de.tudarmstadt.ukp.clarin.webanno.api.DocumentService;
 import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer;
 import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument;
 import de.tudarmstadt.ukp.clarin.webanno.security.UserDao;
@@ -46,13 +45,11 @@
 import de.tudarmstadt.ukp.inception.active.learning.event.ActiveLearningRecommendationEvent;
 import de.tudarmstadt.ukp.inception.active.learning.strategy.ActiveLearningStrategy;
 import de.tudarmstadt.ukp.inception.annotation.layer.span.SpanAdapter;
+import de.tudarmstadt.ukp.inception.documents.api.DocumentService;
 import de.tudarmstadt.ukp.inception.recommendation.api.LearningRecordService;
 import de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService;
 import de.tudarmstadt.ukp.inception.recommendation.api.model.AnnotationSuggestion;
-import de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecord;
-import de.tudarmstadt.ukp.inception.recommendation.api.model.Predictions;
 import de.tudarmstadt.ukp.inception.recommendation.api.model.SpanSuggestion;
-import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionDocumentGroup;
 import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionGroup;
 import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionGroup.Delta;
 import de.tudarmstadt.ukp.inception.schema.AnnotationSchemaService;
@@ -68,7 +65,7 @@
 public class ActiveLearningServiceImpl
     implements ActiveLearningService
 {
-    private final Logger log = LoggerFactory.getLogger(getClass());
+    private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
     private final ApplicationEventPublisher applicationEventPublisher;
     private final DocumentService documentService;
@@ -97,39 +94,20 @@ public ActiveLearningServiceImpl(DocumentService aDocumentService,
     @Override
     public List<SuggestionGroup<SpanSuggestion>> getSuggestions(User aUser, AnnotationLayer aLayer)
     {
-        Predictions predictions = recommendationService.getPredictions(aUser, aLayer.getProject());
+        var predictions = recommendationService.getPredictions(aUser, aLayer.getProject());
 
         if (predictions == null) {
             return emptyList();
         }
 
-        Map<String, SuggestionDocumentGroup<SpanSuggestion>> recommendationsMap = predictions
-                .getPredictionsForWholeProject(SpanSuggestion.class, aLayer, documentService);
+        var recommendationsMap = predictions.getPredictionsForWholeProject(SpanSuggestion.class,
+                aLayer, documentService);
 
         return recommendationsMap.values().stream() //
                 .flatMap(docMap -> docMap.stream()) //
                 .collect(toList());
     }
 
-    @Override
-    public boolean isSuggestionVisible(LearningRecord aRecord)
-    {
-        var aSessionOwner = userService.get(aRecord.getUser());
-        var suggestionGroups = getSuggestions(aSessionOwner, aRecord.getLayer());
-        for (var suggestionGroup : suggestionGroups) {
-            if (suggestionGroup.stream().anyMatch(suggestion -> suggestion.getDocumentName()
-                    .equals(aRecord.getSourceDocument().getName())
-                    && suggestion.getFeature().equals(aRecord.getAnnotationFeature().getName())
-                    && suggestion.labelEquals(aRecord.getAnnotation())
-                    && suggestion.getBegin() == aRecord.getOffsetBegin()
-                    && suggestion.getEnd() == aRecord.getOffsetEnd() //
-                    && suggestion.isVisible())) {
-                return true;
-            }
-        }
-        return false;
-    }
-
     @Override
     public boolean hasSkippedSuggestions(String aSessionOwner, User aDataOwner,
             AnnotationLayer aLayer)
@@ -174,27 +152,31 @@ public Optional<Delta<SpanSuggestion>> generateNextSuggestion(String aSessionOwn
         long startTimer = System.currentTimeMillis();
         var suggestionGroups = alState.getSuggestions();
         long getRecommendationsFromRecommendationService = System.currentTimeMillis();
-        log.trace("Getting recommendations from recommender system took {} ms.",
+        LOG.trace("Getting recommendations from recommender system took {} ms.",
                 (getRecommendationsFromRecommendationService - startTimer));
 
-        // remove duplicate recommendations
-        suggestionGroups = suggestionGroups.stream() //
-                .map(it -> removeDuplicateRecommendations(it)) //
-                .collect(toList());
-        long removeDuplicateRecommendation = System.currentTimeMillis();
-        log.trace("Removing duplicate recommendations took {} ms.",
-                (removeDuplicateRecommendation - getRecommendationsFromRecommendationService));
-
         // hide rejected recommendations
         hideRejectedOrSkippedAnnotations(aSessionOwner, aDataOwner, alState.getLayer(), true,
                 suggestionGroups);
         long removeRejectedSkippedRecommendation = System.currentTimeMillis();
-        log.trace("Removing rejected or skipped ones took {} ms.",
-                (removeRejectedSkippedRecommendation - removeDuplicateRecommendation));
+        LOG.trace("Hiding rejected or skipped ones took {} ms.",
+                (removeRejectedSkippedRecommendation
+                        - getRecommendationsFromRecommendationService));
+
+        // remove duplicate recommendations
+        suggestionGroups = suggestionGroups.stream() //
+                .map(it -> removeDuplicatesAndHiddenSuggestions(it)) //
+                .filter(it -> !it.isEmpty()) //
+                .collect(toList());
+        long removeDuplicateRecommendation = System.currentTimeMillis();
+        LOG.trace("Removing duplicate recommendations took {} ms.",
+                (removeDuplicateRecommendation - removeRejectedSkippedRecommendation));
 
         var pref = recommendationService.getPreferences(aDataOwner,
                 alState.getLayer().getProject());
-        return alState.getStrategy().generateNextSuggestion(pref, suggestionGroups);
+        var nextSuggestion = alState.getStrategy().generateNextSuggestion(pref, suggestionGroups);
+        assert nextSuggestion.get().getFirst().isVisible() : "Generated suggestion must be visible";
+        return nextSuggestion;
     }
 
     @Override
@@ -296,35 +278,40 @@ public void skipSpanSuggestion(String aSessionOwner, User aDataOwner, Annotation
                 alternativeSuggestions));
     }
 
-    private static SuggestionGroup<SpanSuggestion> removeDuplicateRecommendations(
-            SuggestionGroup<SpanSuggestion> unmodifiedRecommendationList)
+    private static SuggestionGroup<SpanSuggestion> removeDuplicatesAndHiddenSuggestions(
+            SuggestionGroup<SpanSuggestion> aSuggestionGroup)
     {
-        SuggestionGroup<SpanSuggestion> cleanRecommendationList = new SuggestionGroup<>();
+        var cleanSuggestionGroup = new SuggestionGroup<SpanSuggestion>();
 
-        unmodifiedRecommendationList.forEach(recommendationItem -> {
-            if (!isAlreadyInCleanList(cleanRecommendationList, recommendationItem)) {
-                cleanRecommendationList.add(recommendationItem);
+        aSuggestionGroup.forEach(suggestion -> {
+            if (!suggestion.isVisible()) {
+                return;
+            }
+
+            if (!isAlreadyInCleanList(cleanSuggestionGroup, suggestion)) {
+                cleanSuggestionGroup.add(suggestion);
             }
         });
 
-        return cleanRecommendationList;
+        return cleanSuggestionGroup;
     }
 
     private static boolean isAlreadyInCleanList(
             SuggestionGroup<SpanSuggestion> cleanRecommendationList,
             AnnotationSuggestion recommendationItem)
     {
-        String source = recommendationItem.getRecommenderName();
-        String annotation = recommendationItem.getLabel();
-        String documentName = recommendationItem.getDocumentName();
+        var source = recommendationItem.getRecommenderName();
+        var annotation = recommendationItem.getLabel();
+        var documentName = recommendationItem.getDocumentName();
 
-        for (AnnotationSuggestion existingRecommendation : cleanRecommendationList) {
-            boolean areLabelsEqual = existingRecommendation.labelEquals(annotation);
+        for (var existingRecommendation : cleanRecommendationList) {
+            var areLabelsEqual = existingRecommendation.labelEquals(annotation);
             if (existingRecommendation.getRecommenderName().equals(source) && areLabelsEqual
                     && existingRecommendation.getDocumentName().equals(documentName)) {
                 return true;
             }
         }
+
         return false;
     }
 

inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/config/ActiveLearningAutoConfiguration.java

@@ -24,13 +24,13 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
-import de.tudarmstadt.ukp.clarin.webanno.api.DocumentService;
 import de.tudarmstadt.ukp.clarin.webanno.security.UserDao;
 import de.tudarmstadt.ukp.inception.active.learning.ActiveLearningService;
 import de.tudarmstadt.ukp.inception.active.learning.ActiveLearningServiceImpl;
 import de.tudarmstadt.ukp.inception.active.learning.log.ActiveLearningRecommendationEventAdapter;
 import de.tudarmstadt.ukp.inception.active.learning.log.ActiveLearningSuggestionOfferedAdapter;
 import de.tudarmstadt.ukp.inception.active.learning.sidebar.ActiveLearningSidebarFactory;
+import de.tudarmstadt.ukp.inception.documents.api.DocumentService;
 import de.tudarmstadt.ukp.inception.recommendation.api.LearningRecordService;
 import de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService;
 import de.tudarmstadt.ukp.inception.recommendation.config.RecommenderServiceAutoConfiguration;