From 351e3ae1fb30a189973156cde3b42934ee1e9136 Mon Sep 17 00:00:00 2001 From: Lixi Date: Wed, 31 Mar 2021 11:04:39 +0200 Subject: [PATCH 1/6] Update README.md --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index 5aff9ca2..6958aed2 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,34 @@ It contains the following files: You can use the [basic configuration](https://github.com/dice-group/IGUANA/blob/master/example-suite.yml) we provide and modify it to your needs. For further information please visit our [configuration](http://iguana-benchmark.eu/docs/3.2/usage/configuration/) and [Stresstest](http://iguana-benchmark.eu/docs/3.0/usage/stresstest/) wiki pages. For a detailed, step-by-step instruction please attend our [tutorial](http://iguana-benchmark.eu/docs/3.2/usage/tutorial/). +# How to Cite + +```bibtex +@InProceedings{10.1007/978-3-319-68204-4_5, +author="Conrads, Felix +and Lehmann, Jens +and Saleem, Muhammad +and Morsey, Mohamed +and Ngonga Ngomo, Axel-Cyrille", +editor="d'Amato, Claudia +and Fernandez, Miriam +and Tamma, Valentina +and Lecue, Freddy +and Cudr{\'e}-Mauroux, Philippe +and Sequeda, Juan +and Lange, Christoph +and Heflin, Jeff", +title="Iguana: A Generic Framework for Benchmarking the Read-Write Performance of Triple Stores", +booktitle="The Semantic Web -- ISWC 2017", +year="2017", +publisher="Springer International Publishing", +address="Cham", +pages="48--65", +abstract="The performance of triples stores is crucial for applications driven by RDF. Several benchmarks have been proposed that assess the performance of triple stores. However, no integrated benchmark-independent execution framework for these benchmarks has yet been provided. We propose a novel SPARQL benchmark execution framework called Iguana. Our framework complements benchmarks by providing an execution environment which can measure the performance of triple stores during data loading, data updates as well as under different loads and parallel requests. Moreover, it allows a uniform comparison of results on different benchmarks. We execute the FEASIBLE and DBPSB benchmarks using the Iguana framework and measure the performance of popular triple stores under updates and parallel user requests. We compare our results (See https://doi.org/10.6084/m9.figshare.c.3767501.v1) with state-of-the-art benchmarking results and show that our benchmark execution framework can unveil new insights pertaining to the performance of triple stores.", +isbn="978-3-319-68204-4" +} +``` + ## Execute the Benchmark Use the start script From 98d0385da97fd8ebf17f85a24c513d26053c1f63 Mon Sep 17 00:00:00 2001 From: Lixi Date: Wed, 31 Mar 2021 11:04:57 +0200 Subject: [PATCH 2/6] Update README.md --- README.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6958aed2..01533b33 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,17 @@ It contains the following files: You can use the [basic configuration](https://github.com/dice-group/IGUANA/blob/master/example-suite.yml) we provide and modify it to your needs. For further information please visit our [configuration](http://iguana-benchmark.eu/docs/3.2/usage/configuration/) and [Stresstest](http://iguana-benchmark.eu/docs/3.0/usage/stresstest/) wiki pages. For a detailed, step-by-step instruction please attend our [tutorial](http://iguana-benchmark.eu/docs/3.2/usage/tutorial/). + + +## Execute the Benchmark + +Use the start script +``` +./start-iguana.sh example-suite.yml +``` +Now Iguana will execute the example benchmark suite configured in the example-suite.yml file + + # How to Cite ```bibtex @@ -135,11 +146,3 @@ abstract="The performance of triples stores is crucial for applications driven b isbn="978-3-319-68204-4" } ``` - -## Execute the Benchmark - -Use the start script -``` -./start-iguana.sh example-suite.yml -``` -Now Iguana will execute the example benchmark suite configured in the example-suite.yml file From 833c3011b805fc8b65944079ae884094d1b4cdac Mon Sep 17 00:00:00 2001 From: Lixi Date: Wed, 31 Mar 2021 18:30:38 +0200 Subject: [PATCH 3/6] Update HttpWorker.java --- .../main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java index 1d4aa5d6..d896c802 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java @@ -136,7 +136,6 @@ synchronized protected void addResultsOnce(QueryExecutionStats queryExecutionSta @Override public void executeQuery(String query, String queryID) { - requestStartTime = Instant.now(); queryId = queryID; resultsSaved = false; requestTimedOut = false; @@ -148,7 +147,8 @@ public void executeQuery(String query, String queryID) { buildRequest(query, queryId); setTimeout(timeOut.intValue()); - + + requestStartTime = Instant.now(); response = client.execute(request, getAuthContext(con.getEndpoint())); // method to process the result in background processHttpResponse(); From 2f03e6c074f671c51a87c40d6f5592cf0061ed0b Mon Sep 17 00:00:00 2001 From: Alexander Bigerl Date: Wed, 7 Apr 2021 22:31:51 +0200 Subject: [PATCH 4/6] proof of concept: fix performance issue --- .../aksw/iguana/commons/streams/Streams.java | 20 +++++++++++++++++++ .../lang/impl/ThrowawayLanguageProcessor.java | 4 +++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java b/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java index ddff712e..e1b379fe 100644 --- a/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java +++ b/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java @@ -109,4 +109,24 @@ static public long inputStream2Length(InputStream inputStream, Instant startTime return ret; } + /** + * reads a stream and throws away the result. + * + * @param inputStream the stream to read from + * @param timeout delta from startTime when the computation must be completed. Otherwise a TimeoutException may be thrown. Timeout check is deactivated if timeout is < 0. + * @return size of the output stream + * @throws IOException from inputStream.read + * @throws TimeoutException Maybe thrown any time after if startTime + timeout is exceed + */ + static public long inputStream2Length(InputStream inputStream, Instant startTime, double timeout, byte[] buffer) throws IOException, TimeoutException { + long length; + long ret = 0; + while ((length = inputStream.read(buffer)) != -1) { + if (durationInMilliseconds(startTime, Instant.now()) > timeout && timeout >0) + throw new TimeoutException("reading the answer timed out"); + ret += length; + } + return ret; + } + } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java index 5f226793..8db8dfeb 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java @@ -16,10 +16,12 @@ @Shorthand("lang.SIMPLE") public class ThrowawayLanguageProcessor extends AbstractLanguageProcessor { + byte[] buffer = new byte[10 * 1024 * 1024]; // 10 MB buffer + @Override public long readResponse(InputStream inputStream, ByteArrayOutputStream responseBody) throws IOException, TimeoutException { - return Streams.inputStream2Length(inputStream, Instant.now(), 0); + return Streams.inputStream2Length(inputStream, Instant.now(), 0, buffer); } @Override From db7fae5e1e70c2e2260fd8ed96f868688ea16998 Mon Sep 17 00:00:00 2001 From: Alexander Bigerl Date: Mon, 12 Apr 2021 11:26:58 +0200 Subject: [PATCH 5/6] reuse buffers on a per-thread level --- .../aksw/iguana/commons/streams/Streams.java | 39 +++++++------------ .../lang/impl/ThrowawayLanguageProcessor.java | 4 +- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java b/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java index e1b379fe..1553bc4b 100644 --- a/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java +++ b/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java @@ -13,6 +13,13 @@ * Helper functions to work with streams. */ public class Streams { + + final static int bufferSize = 16 * 1024 * 1024; // 16 MB buffer + + final static ThreadLocal threadBuffer = ThreadLocal.withInitial(() -> new byte[bufferSize]); + + final static ThreadLocal threadByteArrayOutputStream = ThreadLocal.withInitial(() -> new ByteArrayOutputStream(bufferSize)); + /** * Fastest way to serialize a stream to UTF-8 according to https://stackoverflow.com/a/35446009/6800941 * @@ -21,7 +28,8 @@ public class Streams { * @throws IOException from inputStream.read */ static public ByteArrayOutputStream inputStream2String(InputStream inputStream) throws IOException { - ByteArrayOutputStream result = new ByteArrayOutputStream(); + ByteArrayOutputStream result = threadByteArrayOutputStream.get(); + result.reset(); try { inputStream2ByteArrayOutputStream(inputStream, null, -1.0, result); } catch (TimeoutException e) { @@ -61,7 +69,7 @@ static public ByteArrayOutputStream inputStream2String(InputStream inputStream, public static long inputStream2ByteArrayOutputStream(InputStream inputStream, Instant startTime, double timeout, ByteArrayOutputStream result) throws IOException, TimeoutException { assert (result != null); boolean enable_timeout = timeout > 0; - byte[] buffer = new byte[10 * 1024 * 1024]; // 10 MB buffer + byte[] buffer = threadBuffer.get(); int length; while ((length = inputStream.read(buffer)) != -1) { if (enable_timeout && durationInMilliseconds(startTime, Instant.now()) > timeout) @@ -82,7 +90,7 @@ public static long inputStream2ByteArrayOutputStream(InputStream inputStream, In public static long inputStream2ByteArrayOutputStream(InputStream inputStream, ByteArrayOutputStream result) throws IOException { try { return inputStream2ByteArrayOutputStream(inputStream, Instant.now(), -1, result); - }catch(TimeoutException e){ + } catch (TimeoutException e) { //will never happen return 0; } @@ -98,35 +106,14 @@ public static long inputStream2ByteArrayOutputStream(InputStream inputStream, By * @throws TimeoutException Maybe thrown any time after if startTime + timeout is exceed */ static public long inputStream2Length(InputStream inputStream, Instant startTime, double timeout) throws IOException, TimeoutException { - byte[] buffer = new byte[10 * 1024 * 1024]; // 10 MB buffer + byte[] buffer = threadBuffer.get(); long length; long ret = 0; while ((length = inputStream.read(buffer)) != -1) { - if (durationInMilliseconds(startTime, Instant.now()) > timeout && timeout >0) + if (durationInMilliseconds(startTime, Instant.now()) > timeout && timeout > 0) throw new TimeoutException("reading the answer timed out"); ret += length; } return ret; } - - /** - * reads a stream and throws away the result. - * - * @param inputStream the stream to read from - * @param timeout delta from startTime when the computation must be completed. Otherwise a TimeoutException may be thrown. Timeout check is deactivated if timeout is < 0. - * @return size of the output stream - * @throws IOException from inputStream.read - * @throws TimeoutException Maybe thrown any time after if startTime + timeout is exceed - */ - static public long inputStream2Length(InputStream inputStream, Instant startTime, double timeout, byte[] buffer) throws IOException, TimeoutException { - long length; - long ret = 0; - while ((length = inputStream.read(buffer)) != -1) { - if (durationInMilliseconds(startTime, Instant.now()) > timeout && timeout >0) - throw new TimeoutException("reading the answer timed out"); - ret += length; - } - return ret; - } - } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java index 8db8dfeb..5f226793 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java @@ -16,12 +16,10 @@ @Shorthand("lang.SIMPLE") public class ThrowawayLanguageProcessor extends AbstractLanguageProcessor { - byte[] buffer = new byte[10 * 1024 * 1024]; // 10 MB buffer - @Override public long readResponse(InputStream inputStream, ByteArrayOutputStream responseBody) throws IOException, TimeoutException { - return Streams.inputStream2Length(inputStream, Instant.now(), 0, buffer); + return Streams.inputStream2Length(inputStream, Instant.now(), 0); } @Override From 5b1266eb0d44f5ac1a25d1346a2cf66713e33041 Mon Sep 17 00:00:00 2001 From: Alexander Bigerl Date: Mon, 12 Apr 2021 15:52:24 +0200 Subject: [PATCH 6/6] set visibility of Streams' static members --- .../main/java/org/aksw/iguana/commons/streams/Streams.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java b/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java index 1553bc4b..55ef9edd 100644 --- a/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java +++ b/iguana.commons/src/main/java/org/aksw/iguana/commons/streams/Streams.java @@ -14,11 +14,11 @@ */ public class Streams { - final static int bufferSize = 16 * 1024 * 1024; // 16 MB buffer + public static final int bufferSize = 16 * 1024 * 1024; // 16 MB buffer - final static ThreadLocal threadBuffer = ThreadLocal.withInitial(() -> new byte[bufferSize]); + protected static final ThreadLocal threadBuffer = ThreadLocal.withInitial(() -> new byte[bufferSize]); - final static ThreadLocal threadByteArrayOutputStream = ThreadLocal.withInitial(() -> new ByteArrayOutputStream(bufferSize)); + protected static final ThreadLocal threadByteArrayOutputStream = ThreadLocal.withInitial(() -> new ByteArrayOutputStream(bufferSize)); /** * Fastest way to serialize a stream to UTF-8 according to https://stackoverflow.com/a/35446009/6800941