diff --git a/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java b/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java index 38faf8e7497..89098ca19f8 100644 --- a/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java +++ b/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java @@ -150,7 +150,7 @@ public class ErrorCodes { public static final ErrorCode FODC0001 = new W3CErrorCode("FODC0001", "No context document."); public static final ErrorCode FODC0002 = new W3CErrorCode("FODC0002", "Error retrieving resource."); public static final ErrorCode FODC0003 = new W3CErrorCode("FODC0003", "Function stability not defined."); - public static final ErrorCode FODC0004 = new W3CErrorCode("FODC0004", "Invalid argument to fn:collection."); + public static final ErrorCode FODC0004 = new W3CErrorCode("FODC0004", "Invalid argument to fn:collection or fn:uri-collection."); public static final ErrorCode FODC0005 = new W3CErrorCode("FODC0005", "Invalid argument to fn:doc or fn:doc-available."); public static final ErrorCode FODT0001 = new W3CErrorCode("FODT0001", "Overflow/underflow in date/time operation."); public static final ErrorCode FODT0002 = new W3CErrorCode("FODT0002", "Overflow/underflow in duration operation."); diff --git a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java index 4ebea126e94..502c99d6f8d 100644 --- a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java +++ b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java @@ -203,6 +203,8 @@ public class XQueryContext implements BinaryValueManager, Context { private XMLGregorianCalendar calendar = null; private TimeZone implicitTimeZone = null; + private final Map cachedUriCollectionResults = new HashMap<>(); + /** * the watchdog object assigned to this query. */ @@ -1403,6 +1405,8 @@ public void reset(final boolean keepGlobals) { callStack.clear(); protectedDocuments = null; + cachedUriCollectionResults.clear(); + if (!keepGlobals) { globalVariables.clear(); } @@ -2790,6 +2794,10 @@ public void setStaticDecimalFormat(final QName qnDecimalFormat, final DecimalFor staticDecimalFormats.put(qnDecimalFormat, decimalFormat); } + public Map getCachedUriCollectionResults() { + return cachedUriCollectionResults; + } + /** * Save state */ diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java index aca7ebe1197..06314f9cde7 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java @@ -223,6 +223,8 @@ public class FnModule extends AbstractInternalModule { new FunctionDef(FunTrueOrFalse.fnFalse, FunTrueOrFalse.class), new FunctionDef(FunUpperOrLowerCase.fnLowerCase, FunUpperOrLowerCase.class), new FunctionDef(FunUpperOrLowerCase.fnUpperCase, FunUpperOrLowerCase.class), + new FunctionDef(FunUriCollection.FS_URI_COLLECTION_SIGNATURES[0], FunUriCollection.class), + new FunctionDef(FunUriCollection.FS_URI_COLLECTION_SIGNATURES[1], FunUriCollection.class), new FunctionDef(FunXmlToJson.FS_XML_TO_JSON[0], FunXmlToJson.class), new FunctionDef(FunXmlToJson.FS_XML_TO_JSON[1], FunXmlToJson.class), new FunctionDef(FunZeroOrOne.signature, FunZeroOrOne.class), diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunUriCollection.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunUriCollection.java new file mode 100644 index 00000000000..028225148a0 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunUriCollection.java @@ -0,0 +1,240 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.exist.xquery.functions.fn; + +import org.exist.collections.Collection; +import org.exist.dom.persistent.BinaryDocument; +import org.exist.dom.persistent.DocumentImpl; +import org.exist.security.PermissionDeniedException; +import org.exist.storage.lock.Lock; +import org.exist.util.LockException; +import org.exist.util.PatternFactory; +import org.exist.xmldb.XmldbURI; +import org.exist.xquery.*; +import org.exist.xquery.value.*; + +import java.net.URISyntaxException; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.exist.xquery.FunctionDSL.*; +import static org.exist.xquery.functions.fn.FnModule.functionSignatures; + +public class FunUriCollection extends BasicFunction { + + private static final String FN_NAME = "uri-collection"; + private static final String FN_DESCRIPTION = "Returns a sequence of xs:anyURI values that represent the URIs in a URI collection."; + private static final FunctionReturnSequenceType FN_RETURN = returnsOptMany(Type.ANY_URI, + "the default URI collection, if $arg is not specified or is an empty sequence, " + + "or the sequence of URIs that correspond to the supplied URI"); + private static final FunctionParameterSequenceType ARG = optParam("arg", Type.STRING, + "An xs:string identifying a URI Collection. " + + "The argument is interpreted as either an absolute xs:anyURI, or a relative xs:anyURI resolved " + + "against the base-URI property from the static context. In eXist-db this function consults the " + + "query hierarchy of the database. Query String parameters may be provided to " + + "control the URIs returned by this function. " + + "The parameter `match` may be used to provide a Regular Expression against which the result " + + "sequence of URIs are filtered. " + + "The parameter `content-type` may be used to determine the Internet Media Type (or generally " + + "whether XML, Binary, and/or (Sub) Collection) URIs that are returned in the result sequence; " + + "the special values: 'application/vnd.existdb.collection' includes (Sub) Collections, " + + "'application/vnd.existdb.document' includes any document, " + + "'application/vnd.existdb.document+xml' includes only XML documents, and " + + "'application/vnd.existdb.document+binary' includes only Binary documents. By default, " + + "`content-type=application/vnd.existdb.collection,application/vnd.existdb.document` " + + "(i.e. all Collections and Documents). " + + "The parameter `stable` may be used to determine if the function is deterministic. " + + "By default `stable=yes` to ensure that the same results are returned by each call within the same " + + "query." + ); + public static final FunctionSignature[] FS_URI_COLLECTION_SIGNATURES = functionSignatures( + FN_NAME, + FN_DESCRIPTION, + FN_RETURN, + arities( + arity(), + arity(ARG) + ) + ); + + private static final String KEY_CONTENT_TYPE = "content-type"; + private static final String VALUE_CONTENT_TYPE_DOCUMENT = "application/vnd.existdb.document"; + private static final String VALUE_CONTENT_TYPE_DOCUMENT_BINARY = "application/vnd.existdb.document+binary"; + private static final String VALUE_CONTENT_TYPE_DOCUMENT_XML = "application/vnd.existdb.document+xml"; + private static final String VALUE_CONTENT_TYPE_SUBCOLLECTION = "application/vnd.existdb.collection"; + private static final String[] VALUE_CONTENT_TYPES = { + VALUE_CONTENT_TYPE_DOCUMENT, + VALUE_CONTENT_TYPE_DOCUMENT_BINARY, + VALUE_CONTENT_TYPE_DOCUMENT_XML, + VALUE_CONTENT_TYPE_SUBCOLLECTION + }; + + private static final String KEY_STABLE = "stable"; + private static final String VALUE_STABLE_NO = "no"; + private static final String VALUE_STABLE_YES = "yes"; + private static final String[] VALUE_STABLES = { + VALUE_STABLE_NO, + VALUE_STABLE_YES + }; + + private static final String KEY_MATCH = "match"; + + public FunUriCollection(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence result; + if (args.length == 0 || args[0].isEmpty() || args[0].toString().isEmpty()) { + result = new AnyURIValue(XmldbURI.ROOT_COLLECTION); + } else { + final List resultUris = new ArrayList<>(); + + final String uriWithQueryString = args[0].toString(); + final int queryStringIndex = uriWithQueryString.indexOf('?'); + final String uriWithoutQueryString = (queryStringIndex >= 0) ? uriWithQueryString.substring(0, queryStringIndex) : uriWithQueryString; + String uriWithoutStableQueryString = uriWithQueryString.replaceAll(String.format("%s\\s*=\\s*\\byes|no\\b\\s*&+", KEY_STABLE), ""); + if (uriWithoutStableQueryString.endsWith("?")) { + uriWithoutStableQueryString = uriWithoutStableQueryString.substring(0, uriWithoutStableQueryString.length() - 1); + } + + final XmldbURI uri; + try { + uri = XmldbURI.xmldbUriFor(uriWithoutQueryString); + } catch (URISyntaxException e) { + throw new XPathException(this, ErrorCodes.FODC0004, String.format("\"%s\" is not a valid URI.", args[0].toString())); + } + + final Map queryStringMap = parseQueryString(uriWithQueryString); + checkQueryStringMap(queryStringMap); + + if ((!queryStringMap.containsKey(KEY_STABLE) || queryStringMap.get(KEY_STABLE).equals(VALUE_STABLE_YES)) && + context.getCachedUriCollectionResults().containsKey(uriWithoutStableQueryString)) { + result = context.getCachedUriCollectionResults().get(uriWithoutStableQueryString); + } else { + final boolean binaryUrisIncluded = !queryStringMap.containsKey(KEY_CONTENT_TYPE) || + (queryStringMap.get(KEY_CONTENT_TYPE).equals(VALUE_CONTENT_TYPE_DOCUMENT) || + queryStringMap.get(KEY_CONTENT_TYPE).equals(VALUE_CONTENT_TYPE_DOCUMENT_BINARY)); + final boolean subcollectionUrisIncluded = !queryStringMap.containsKey(KEY_CONTENT_TYPE) || + queryStringMap.get(KEY_CONTENT_TYPE).equals(VALUE_CONTENT_TYPE_SUBCOLLECTION); + final boolean xmlUrisIncluded = !queryStringMap.containsKey(KEY_CONTENT_TYPE) || + (queryStringMap.get(KEY_CONTENT_TYPE).equals(VALUE_CONTENT_TYPE_DOCUMENT) || + queryStringMap.get(KEY_CONTENT_TYPE).equals(VALUE_CONTENT_TYPE_DOCUMENT_XML)); + + try (final Collection collection = context.getBroker().openCollection(uri, Lock.LockMode.READ_LOCK)) { + if (collection != null) { + if (binaryUrisIncluded || xmlUrisIncluded) { + final Iterator documentIterator = collection.iterator(context.getBroker()); + while (documentIterator.hasNext()) { + final DocumentImpl document = documentIterator.next(); + if ((xmlUrisIncluded && !(document instanceof BinaryDocument)) || + (binaryUrisIncluded && document instanceof BinaryDocument)) { + resultUris.add(document.getURI().toString()); + } + } + } + + if (subcollectionUrisIncluded) { + final Iterator collectionsIterator = collection.collectionIterator(context.getBroker()); + while (collectionsIterator.hasNext()) { + resultUris.add(uri.append(collectionsIterator.next()).toString()); + } + } + } else { + throw new XPathException(this, ErrorCodes.FODC0002, String.format("Collection \"%s\" not found.", uri)); + } + } catch (final LockException | PermissionDeniedException e) { + throw new XPathException(this, ErrorCodes.FODC0002, e); + } + + if (queryStringMap.containsKey(KEY_MATCH) && queryStringMap.get(KEY_MATCH).length() > 0) { + final Pattern pattern = PatternFactory.getInstance().getPattern(queryStringMap.get(KEY_MATCH)); + final List matchedResultUris = resultUris.stream().filter(resultUri -> pattern.matcher(resultUri).find()).collect(Collectors.toList()); + if (matchedResultUris.isEmpty()) { + result = Sequence.EMPTY_SEQUENCE; + } else { + result = new ValueSequence(); + for (String resultUri : matchedResultUris) { + result.add(new AnyURIValue(resultUri)); + } + } + } else { + result = new ValueSequence(); + for (String resultUri : resultUris) { + result.add(new AnyURIValue(resultUri)); + } + } + + // only store the result if they were not previously stored - otherwise we loose stability! + if (!context.getCachedUriCollectionResults().containsKey(uriWithoutStableQueryString)) { + context.getCachedUriCollectionResults().put(uriWithoutStableQueryString, result); + } + } + } + + return result; + } + + private static Map parseQueryString(final String uri) { + final Map map = new HashMap<>(); + if (uri != null) { + final int questionMarkIndex = uri.indexOf('?'); + if (questionMarkIndex >= 0 && questionMarkIndex + 1 < uri.length()) { + String[] keyValuePairs = uri.substring(questionMarkIndex + 1).split("&"); + for (String keyValuePair : keyValuePairs) { + int equalIndex = keyValuePair.indexOf('='); + if (equalIndex >= 0) { + if (equalIndex + 1 < uri.length()) { + map.put(keyValuePair.substring(0, equalIndex).trim(), keyValuePair.substring(equalIndex + 1).trim()); + } else { + map.put(keyValuePair.substring(0, equalIndex).trim(), ""); + } + } else { + map.put(keyValuePair.trim(), ""); + } + } + } + } + + return map; + } + + private void checkQueryStringMap(final Map queryStringMap) throws XPathException { + for (Map.Entry queryStringEntry : queryStringMap.entrySet()) { + final String key = queryStringEntry.getKey(); + final String value = queryStringEntry.getValue(); + if (key.equals(KEY_CONTENT_TYPE)) { + if (Arrays.stream(VALUE_CONTENT_TYPES).noneMatch(contentTypeValue -> contentTypeValue.equals(value))) { + throw new XPathException(this, ErrorCodes.FODC0004, String.format("Invalid query-string value \"%s\".", queryStringEntry)); + } + } else if (key.equals(KEY_STABLE)) { + if (Arrays.stream(VALUE_STABLES).noneMatch(stableValue -> stableValue.equals(value))) { + throw new XPathException(this, ErrorCodes.FODC0004, String.format("Invalid query-string value \"%s\".", queryStringEntry)); + } + } else if (!key.equals(KEY_MATCH)) { + throw new XPathException(this, ErrorCodes.FODC0004, String.format("Unexpected query string \"%s\".", queryStringEntry)); + } + } + } +} diff --git a/exist-core/src/test/xquery/xquery3/fnUriCollection.xqm b/exist-core/src/test/xquery/xquery3/fnUriCollection.xqm new file mode 100644 index 00000000000..055299bd98c --- /dev/null +++ b/exist-core/src/test/xquery/xquery3/fnUriCollection.xqm @@ -0,0 +1,133 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +module namespace fnuc="http://exist-db.org/xquery/test/function_uri_collection"; + +declare namespace test="http://exist-db.org/xquery/xqsuite"; +declare namespace x="httpx://x/ns/1.0"; + +declare variable $fnuc:COLLECTION_NAME := "/test-collection"; +declare variable $fnuc:SUBCOLLECTION_NAME := $fnuc:COLLECTION_NAME||"/subcol"; +declare variable $fnuc:COLLECTION := collection("/db"||$fnuc:COLLECTION_NAME); +declare variable $fnuc:SUBCOLLECTION := collection("/db"||$fnuc:SUBCOLLECTION_NAME); + +declare + %test:setUp +function fnuc:setup() { + let $collection := xmldb:create-collection("/db", "test-collection") + + return + ( + xmldb:create-collection("/db", $fnuc:SUBCOLLECTION_NAME), + xmldb:store("/db"||$fnuc:SUBCOLLECTION_NAME, "test-subcol.xml", ), + xmldb:store($collection, "test.bin", "binary", "application/octet-stream"), + xmldb:store($collection, "test.xml", document { }) + ) +}; + +declare + %test:tearDown +function fnuc:cleanup() { + xmldb:remove("/db/test-collection") +}; + +declare + %test:assertEquals("/db") +function fnuc:no-argument() { + fn:uri-collection() +}; + +declare + %test:assertError("FODC0004") +function fnuc:invalid-uri() { + fn:uri-collection(":invalid-uri") +}; + +declare + %test:assertEquals("/db/test-collection/test.bin", "/db/test-collection/test.xml", "/db/test-collection/subcol") +function fnuc:all-uris() { + fn:uri-collection("/db/test-collection") +}; + +declare + %test:assertEquals("/db/test-collection/subcol") +function fnuc:subcollection-uris() { + fn:uri-collection("/db/test-collection?content-type=application/vnd.existdb.collection") +}; + +declare + %test:assertEquals("/db/test-collection/test.bin", "/db/test-collection/test.xml") +function fnuc:document-uris() { + fn:uri-collection("/db/test-collection?content-type=application/vnd.existdb.document") +}; + +declare + %test:assertEquals("/db/test-collection/test.xml") +function fnuc:xml-document-uris() { + fn:uri-collection("/db/test-collection?content-type=application/vnd.existdb.document+xml") +}; + +declare + %test:assertEquals("/db/test-collection/test.bin") +function fnuc:binary-document-uris() { + fn:uri-collection("/db/test-collection?content-type=application/vnd.existdb.document+binary") +}; + +declare + %test:assertEquals("/db/test-collection/test.bin") +function fnuc:match-uris() { + fn:uri-collection("/db/test-collection?match=.*\.bin") +}; + +declare + %test:assertEmpty +function fnuc:no-match-uris() { + fn:uri-collection("/db/test-collection?match=.*\.nonexisting") +}; + +declare + %test:assertEquals("/db/test-collection/test.bin", "/db/test-collection/test.xml", "/db/test-collection/subcol") +function fnuc:stable() { + let $c1 := fn:uri-collection("/db/test-collection?stable=yes") + let $r := xmldb:remove("/db/test-collection", "test.xml") + let $c2 := fn:uri-collection("/db/test-collection?stable=yes") + let $a := xmldb:store("/db/test-collection", "test.xml", document { }) + return $c2 +}; + +declare + %test:assertEquals("/db/test-collection/test.bin", "/db/test-collection/subcol") +function fnuc:not-stable() { + let $c1 := fn:uri-collection("/db/test-collection?stable=no") + let $r := xmldb:remove("/db/test-collection", "test.xml") + let $c2 := fn:uri-collection("/db/test-collection?stable=no") + let $a := xmldb:store("/db/test-collection", "test.xml", document { }) + return $c2 +}; + +declare + %test:assertError("FODC0002") +function fnuc:non-existent-resource() { + fn:uri-collection("thisfileshouldnotexists") +}; +