Enable paging for /submodel/submodel-elements/$path (#920)

* fix paging for GetAllSubmodelElementsPath
* update changelog

core/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/request/handler/submodel/GetAllSubmodelElementsPathRequestHandler.java

@@ -16,19 +16,25 @@
 
 import de.fraunhofer.iosb.ilt.faaast.service.assetconnection.AssetConnectionException;
 import de.fraunhofer.iosb.ilt.faaast.service.exception.MessageBusException;
+import de.fraunhofer.iosb.ilt.faaast.service.model.IdShortPath;
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.paging.Page;
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.paging.PagingInfo;
+import de.fraunhofer.iosb.ilt.faaast.service.model.api.paging.PagingMetadata;
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.request.submodel.GetAllSubmodelElementsPathRequest;
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.response.submodel.GetAllSubmodelElementsPathResponse;
 import de.fraunhofer.iosb.ilt.faaast.service.model.exception.ResourceNotAContainerElementException;
 import de.fraunhofer.iosb.ilt.faaast.service.model.exception.ResourceNotFoundException;
 import de.fraunhofer.iosb.ilt.faaast.service.model.exception.ValueMappingException;
 import de.fraunhofer.iosb.ilt.faaast.service.model.messagebus.event.access.ElementReadEventMessage;
+import de.fraunhofer.iosb.ilt.faaast.service.model.visitor.ReferenceCollector;
 import de.fraunhofer.iosb.ilt.faaast.service.request.handler.AbstractSubmodelInterfaceRequestHandler;
 import de.fraunhofer.iosb.ilt.faaast.service.request.handler.RequestExecutionContext;
 import de.fraunhofer.iosb.ilt.faaast.service.util.LambdaExceptionHelper;
 import de.fraunhofer.iosb.ilt.faaast.service.util.ReferenceBuilder;
+import java.util.List;
 import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.eclipse.digitaltwin.aas4j.v3.dataformat.core.util.AasUtils;
 import org.eclipse.digitaltwin.aas4j.v3.model.Reference;
 import org.eclipse.digitaltwin.aas4j.v3.model.SubmodelElement;
@@ -48,21 +54,77 @@ public GetAllSubmodelElementsPathRequestHandler(RequestExecutionContext context)
     }
 
 
+    private static <T> Page<T> preparePagedResult(Stream<T> input, PagingInfo paging) {
+        Stream<T> result = input;
+        if (Objects.nonNull(paging.getCursor())) {
+            result = result.skip(readCursor(paging.getCursor()));
+        }
+        if (paging.hasLimit()) {
+            result = result.limit(paging.getLimit() + 1);
+        }
+        List<T> temp = result.collect(Collectors.toList());
+        return Page.<T> builder()
+                .result(temp.stream()
+                        .limit(paging.hasLimit() ? paging.getLimit() : temp.size())
+                        .collect(Collectors.toList()))
+                .metadata(PagingMetadata.builder()
+                        .cursor(nextCursor(paging, temp.size()))
+                        .build())
+                .build();
+    }
+
+
+    private static long readCursor(String cursor) {
+        return Long.parseLong(cursor);
+    }
+
+
+    private static String writeCursor(long index) {
+        return Long.toString(index);
+    }
+
+
+    private static String nextCursor(PagingInfo paging, boolean hasMoreData) {
+        if (!hasMoreData) {
+            return null;
+        }
+        if (!paging.hasLimit()) {
+            throw new IllegalStateException("unable to generate next cursor for paging - there should not be more data available if previous request did not have a limit set");
+        }
+        if (Objects.isNull(paging.getCursor())) {
+            return writeCursor(paging.getLimit());
+        }
+        return writeCursor(readCursor(paging.getCursor()) + paging.getLimit());
+    }
+
+
+    private static String nextCursor(PagingInfo paging, int resultCount) {
+        return nextCursor(paging, paging.hasLimit() && resultCount > paging.getLimit());
+    }
+
+
     @Override
     public GetAllSubmodelElementsPathResponse doProcess(GetAllSubmodelElementsPathRequest request)
             throws AssetConnectionException, ValueMappingException, ResourceNotFoundException, MessageBusException, ResourceNotAContainerElementException {
         Reference reference = ReferenceBuilder.forSubmodel(request.getSubmodelId());
-        Page<SubmodelElement> page = context.getPersistence().getSubmodelElements(reference, request.getOutputModifier(), PagingInfo.ALL);
-        syncWithAsset(reference, page.getContent(), !request.isInternal());
-        if (!request.isInternal() && Objects.nonNull(page.getContent())) {
-            page.getContent().forEach(LambdaExceptionHelper.rethrowConsumer(
+        Page<SubmodelElement> submodelElements = context.getPersistence().getSubmodelElements(reference, request.getOutputModifier(), PagingInfo.ALL);
+        Page<IdShortPath> page;
+        page = preparePagedResult(submodelElements.getContent().stream()
+                .flatMap(x -> ReferenceCollector.collect(x).keySet().stream()
+                        .map(y -> IdShortPath.combine(
+                                IdShortPath.builder().idShort(x.getIdShort()).build(),
+                                IdShortPath.fromReference(y))))
+                .sorted((x, y) -> x.toString().compareTo(y.toString())),
+                request.getPagingInfo());
+        if (!request.isInternal() && Objects.nonNull(submodelElements.getContent())) {
+            submodelElements.getContent().forEach(LambdaExceptionHelper.rethrowConsumer(
                     x -> context.getMessageBus().publish(ElementReadEventMessage.builder()
                             .element(AasUtils.toReference(reference, x))
                             .value(x)
                             .build())));
         }
         return GetAllSubmodelElementsPathResponse.builder()
-                .payload(page.getContent())
+                .payload(page)
                 .success()
                 .build();
     }

docs/source/other/release-notes.md

@@ -22,6 +22,7 @@
 		- Added support for query parameters `assetId` and `idShort` for `/shells/$reference`
 		- `/serialization` no longer fails when model contains 'embedded' files but target file format does not. Instead, files are ignored.
 		- Fixed bug where `/aas/asset-information/thumbnail` incorrectly returns `500 Internal Server Error` when property `content-type` is not set
+		- Enable paging for `/submodel/submodel-elements/$path`
 - Serialization
 	- JSON
 		- Fixed exception occuring when trying to serialize an element as valueOnly that contains elements that do not support valueOnly serialization

endpoint/http/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/endpoint/http/request/mapper/submodel/GetAllSubmodelElementsPathRequestMapper.java

@@ -17,10 +17,11 @@
 import de.fraunhofer.iosb.ilt.faaast.service.ServiceContext;
 import de.fraunhofer.iosb.ilt.faaast.service.endpoint.http.model.HttpMethod;
 import de.fraunhofer.iosb.ilt.faaast.service.endpoint.http.model.HttpRequest;
-import de.fraunhofer.iosb.ilt.faaast.service.endpoint.http.request.mapper.AbstractSubmodelInterfaceRequestMapper;
+import de.fraunhofer.iosb.ilt.faaast.service.endpoint.http.request.mapper.AbstractSubmodelInterfaceRequestMapperWithPaging;
 import de.fraunhofer.iosb.ilt.faaast.service.endpoint.http.request.mapper.QueryParameters;
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.modifier.Content;
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.modifier.OutputModifier;
+import de.fraunhofer.iosb.ilt.faaast.service.model.api.paging.PagingInfo;
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.request.submodel.GetAllSubmodelElementsPathRequest;
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.response.submodel.GetAllSubmodelElementsPathResponse;
 import java.util.Map;
@@ -31,7 +32,7 @@
  * shells/{aasIdentifier}/submodels/{submodelIdentifier}/submodel-elements/$path.
  */
 public class GetAllSubmodelElementsPathRequestMapper
-        extends AbstractSubmodelInterfaceRequestMapper<GetAllSubmodelElementsPathRequest, GetAllSubmodelElementsPathResponse> {
+        extends AbstractSubmodelInterfaceRequestMapperWithPaging<GetAllSubmodelElementsPathRequest, GetAllSubmodelElementsPathResponse> {
 
     private static final String PATTERN = "submodel-elements/\\$path";
 
@@ -47,7 +48,7 @@ public boolean matchesUrl(HttpRequest httpRequest) {
 
 
     @Override
-    public GetAllSubmodelElementsPathRequest doParse(HttpRequest httpRequest, Map<String, String> urlParameters, OutputModifier outputModifier) {
+    public GetAllSubmodelElementsPathRequest doParse(HttpRequest httpRequest, Map<String, String> urlParameters, OutputModifier outputModifier, PagingInfo pagingInfo) {
         return GetAllSubmodelElementsPathRequest.builder()
                 .build();
     }

endpoint/http/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/endpoint/http/response/mapper/GetAllSubmodelElementsPathResponseMapper.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2021 Fraunhofer IOSB, eine rechtlich nicht selbstaendige
+ * Einrichtung der Fraunhofer-Gesellschaft zur Foerderung der angewandten
+ * Forschung e.V.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package de.fraunhofer.iosb.ilt.faaast.service.endpoint.http.response.mapper;
+
+import de.fraunhofer.iosb.ilt.faaast.service.ServiceContext;
+import de.fraunhofer.iosb.ilt.faaast.service.dataformat.SerializationException;
+import de.fraunhofer.iosb.ilt.faaast.service.endpoint.http.serialization.HttpJsonApiSerializer;
+import de.fraunhofer.iosb.ilt.faaast.service.endpoint.http.util.HttpHelper;
+import de.fraunhofer.iosb.ilt.faaast.service.model.api.MessageType;
+import de.fraunhofer.iosb.ilt.faaast.service.model.api.Result;
+import de.fraunhofer.iosb.ilt.faaast.service.model.api.StatusCode;
+import de.fraunhofer.iosb.ilt.faaast.service.model.api.paging.Page;
+import de.fraunhofer.iosb.ilt.faaast.service.model.api.request.submodel.GetAllSubmodelElementsPathRequest;
+import de.fraunhofer.iosb.ilt.faaast.service.model.api.response.submodel.GetAllSubmodelElementsPathResponse;
+import de.fraunhofer.iosb.ilt.faaast.service.model.exception.InvalidRequestException;
+import jakarta.servlet.http.HttpServletResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * HTTP response mapper for {@link GetAllSubmodelElementsPathResponse}.
+ */
+public class GetAllSubmodelElementsPathResponseMapper extends AbstractResponseMapper<GetAllSubmodelElementsPathResponse, GetAllSubmodelElementsPathRequest> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(GetAllSubmodelElementsPathResponseMapper.class);
+
+    public GetAllSubmodelElementsPathResponseMapper(ServiceContext serviceContext) {
+        super(serviceContext);
+    }
+
+
+    @Override
+    public void map(GetAllSubmodelElementsPathRequest apiRequest, GetAllSubmodelElementsPathResponse apiResponse, HttpServletResponse httpResponse) throws InvalidRequestException {
+        Page<String> result = Page.of(
+                apiResponse.getPayload().getContent().stream().map(Object::toString).toList(),
+                apiResponse.getPayload().getMetadata());
+        try {
+            HttpHelper.sendJson(httpResponse,
+                    apiResponse.getStatusCode(),
+                    new HttpJsonApiSerializer().write(result));
+        }
+        catch (SerializationException e) {
+            LOGGER.warn("error serializing response", e);
+            HttpHelper.send(
+                    httpResponse,
+                    StatusCode.SERVER_INTERNAL_ERROR,
+                    Result.builder()
+                            .message(MessageType.EXCEPTION, e.getMessage())
+                            .build());
+        }
+    }
+}

model/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/model/IdShortPath.java

@@ -73,13 +73,15 @@ public static IdShortPath fromReference(Reference reference) {
         Ensure.require(Objects.nonNull(reference.getKeys()) && !reference.getKeys().isEmpty(), "reference must contain at least one keys");
         Ensure.require(Objects.equals(reference.getType(), ReferenceTypes.MODEL_REFERENCE), "reference must be a model reference");
         int startIndex = 0;
-        if (ReferenceHelper.isKeyType(reference.getKeys().get(0), AssetAdministrationShell.class)) {
-            startIndex = 1;
+        if (ReferenceHelper.isKeyType(reference.getKeys().get(startIndex), AssetAdministrationShell.class)) {
+            startIndex++;
+        }
+        if (ReferenceHelper.isKeyType(reference.getKeys().get(startIndex), Submodel.class)) {
+            startIndex++;
         }
-        ReferenceHelper.ensureKeyType(reference.getKeys().get(startIndex), Submodel.class);
         IdShortPath.Builder builder = IdShortPath.builder();
         boolean inList = false;
-        for (int i = startIndex + 1; i < reference.getKeys().size(); i++) {
+        for (int i = startIndex; i < reference.getKeys().size(); i++) {
             ReferenceHelper.ensureKeyType(reference.getKeys().get(i), SubmodelElement.class);
             if (inList) {
                 builder.index(Long.parseUnsignedLong(reference.getKeys().get(i).getValue()));

model/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/model/api/request/submodel/GetAllSubmodelElementsPathRequest.java

@@ -16,6 +16,7 @@
 
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.modifier.Content;
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.request.AbstractSubmodelInterfaceRequest;
+import de.fraunhofer.iosb.ilt.faaast.service.model.api.request.AbstractSubmodelInterfaceRequestWithPaging;
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.request.OutputModifierConstraints;
 import de.fraunhofer.iosb.ilt.faaast.service.model.api.response.submodel.GetAllSubmodelElementsPathResponse;
 import java.util.Objects;
@@ -24,7 +25,7 @@
 /**
  * Request class for GetAllSubmodelElements requests with content modifier Path.
  */
-public class GetAllSubmodelElementsPathRequest extends AbstractSubmodelInterfaceRequest<GetAllSubmodelElementsPathResponse> {
+public class GetAllSubmodelElementsPathRequest extends AbstractSubmodelInterfaceRequestWithPaging<GetAllSubmodelElementsPathResponse> {
 
     public GetAllSubmodelElementsPathRequest() {
         super(OutputModifierConstraints.builder()

model/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/model/api/response/submodel/GetAllSubmodelElementsPathResponse.java

@@ -14,21 +14,20 @@
  */
 package de.fraunhofer.iosb.ilt.faaast.service.model.api.response.submodel;
 
-import de.fraunhofer.iosb.ilt.faaast.service.model.api.response.AbstractResponseWithPayload;
-import java.util.List;
-import org.eclipse.digitaltwin.aas4j.v3.model.SubmodelElement;
+import de.fraunhofer.iosb.ilt.faaast.service.model.IdShortPath;
+import de.fraunhofer.iosb.ilt.faaast.service.model.api.response.AbstractPagedResponse;
 
 
 /**
  * Response class for GetAllSubmodelElements requests with content modifier Path.
  */
-public class GetAllSubmodelElementsPathResponse extends AbstractResponseWithPayload<List<SubmodelElement>> {
+public class GetAllSubmodelElementsPathResponse extends AbstractPagedResponse<IdShortPath> {
 
     public static Builder builder() {
         return new Builder();
     }
 
-    public static class Builder extends AbstractBuilder<List<SubmodelElement>, GetAllSubmodelElementsPathResponse, Builder> {
+    public static class Builder extends AbstractBuilder<IdShortPath, GetAllSubmodelElementsPathResponse, Builder> {
 
         @Override
         protected Builder getSelf() {