Skip to content

Commit

Permalink
v3.3 prep, closing #139, #140
Browse files Browse the repository at this point in the history
  • Loading branch information
TortugaAttack committed Apr 20, 2021
1 parent 154d9ed commit f0953c7
Show file tree
Hide file tree
Showing 17 changed files with 121 additions and 16 deletions.
2 changes: 1 addition & 1 deletion iguana.commons/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.aksw</groupId>
<artifactId>iguana.commons</artifactId>
<version>3.2.1</version>
<version>3.3.0</version>
<name>Iguana Commons</name>
<description>Iguana Common Classes and Methods</description>
<licenses>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,4 +108,5 @@ public class COMMON {
public static final String PROP_BASE_URI = BASE_URI+"/properties/";
public static final String CLASS_BASE_URI = BASE_URI+"/class/";
public static final String PENALTY = "penalty";
public static final String CONNECTION_VERSION_KEY = "connectionVersion";
}
6 changes: 3 additions & 3 deletions iguana.corecontroller/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.aksw</groupId>
<artifactId>iguana.corecontroller</artifactId>
<version>3.2.1</version>
<version>3.3.0</version>
<name>Iguanas Core Controller</name>
<description>The Controller of Iguanas Core Module. Handeling the messaging and is coordinating the ResultProcessor as well as the dataGenerator. Will be communicating with the Web Controller Module.</description>
<licenses>
Expand Down Expand Up @@ -82,7 +82,7 @@
<dependency>
<groupId>org.aksw</groupId>
<artifactId>iguana.resultprocessor</artifactId>
<version>3.2.1</version>
<version>3.3.0</version>
<exclusions>
<exclusion>
<groupId>org.aksw</groupId>
Expand All @@ -94,7 +94,7 @@
<dependency>
<groupId>org.aksw</groupId>
<artifactId>iguana.commons</artifactId>
<version>3.2.1</version>
<version>3.3.0</version>
</dependency>


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ public void start() throws ExecuteException, IOException {
LOGGER.info("Executing preScriptHook");
String execScript = preScriptHook.replace("{{dataset.name}}", dataset.getName())
.replace("{{connection}}", con.getName())
.replace("{{connection.version}}", con.getVersion("{{connection.version}}"))
.replace("{{taskID}}", taskID+"");
LOGGER.info("Finished preScriptHook");
if(dataset.getFile()!=null){
Expand All @@ -99,6 +100,7 @@ public void start() throws ExecuteException, IOException {
if(postScriptHook!=null){
String execScript = postScriptHook.replace("{{dataset.name}}", dataset.getName())
.replace("{{connection}}", con.getName())
.replace("{{connection.version}}", con.getVersion("{{connection.version}}"))
.replace("{{taskID}}", taskID+"");
if(dataset.getFile()!=null){
execScript = execScript.replace("{{dataset.file}}", dataset.getFile());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,22 @@ public class Connection implements Serializable {
private String endpoint;
@JsonProperty(required = false)
private String updateEndpoint;
@JsonProperty(required = false)
private String version;

public String getVersion() {
return version;
}

public String getVersion(String defaultValue) {
if(version!=null)
return version;
return defaultValue;
}

public void setVersion(String version) {
this.version = version;
}

public String getName() {
return name;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ public void addMetaData() {
metaData.setProperty(COMMON.SUITE_ID_KEY, this.suiteID);
metaData.setProperty(COMMON.DATASET_ID_KEY, this.datasetID);
metaData.setProperty(COMMON.CONNECTION_ID_KEY, this.conID);
if(this.con.getVersion()!=null) {
metaData.setProperty(COMMON.CONNECTION_VERSION_KEY, this.con.getVersion());
}
String className=this.getClass().getCanonicalName();
if(this.getClass().isAnnotationPresent(Shorthand.class)){
className = this.getClass().getAnnotation(Shorthand.class).value();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public void countLinesSuccessfulTest() throws IOException, InterruptedException
//create
Process p = CLIProcessManager.createProcess("echo \"abc\"; wait 100; echo \"t\nt\nabc: test ended suffix\"; wait 1m;");
//count Lines until "test ended" occured
Thread.sleep(50);
Thread.sleep(100);
assertTrue(CLIProcessManager.isReaderReady(p));

assertEquals(3, CLIProcessManager.countLinesUntilStringOccurs(p, "test ended", "failed"));
Expand All @@ -42,7 +42,7 @@ public void countLinesSuccessfulTest() throws IOException, InterruptedException
public void countLinesFailTest() throws IOException, InterruptedException {
//create
Process p = CLIProcessManager.createProcess("echo \"abc\"; wait 100; echo \"abc: failed suffix\"; wait 1m;");
Thread.sleep(50);
Thread.sleep(100);
assertTrue(CLIProcessManager.isReaderReady(p));
//count Lines until "test ended" occured
try{
Expand Down
4 changes: 2 additions & 2 deletions iguana.resultprocessor/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.aksw</groupId>
<artifactId>iguana.resultprocessor</artifactId>
<version>3.2.1</version>
<version>3.3.0</version>
<name>Iguana ResultProcessor</name>
<description>Processing, aggregating and store results from Iguanas core.</description>
<licenses>
Expand Down Expand Up @@ -67,7 +67,7 @@
<dependency>
<groupId>org.aksw</groupId>
<artifactId>iguana.commons</artifactId>
<version>3.2.1</version>
<version>3.3.0</version>
</dependency>
</dependencies>

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.aksw.iguana.rp.metrics.impl;

import org.aksw.iguana.commons.annotation.Shorthand;
import org.aksw.iguana.commons.constants.COMMON;
import org.apache.jena.rdf.model.*;

import java.util.HashMap;
Expand All @@ -19,6 +20,15 @@ public AvgQPSMetric() {
"Will calculate the overall average queries Per second. Further on it will save the totaltime of each query, the failure and the success");
}

public AvgQPSMetric(Integer penalty) {
super(
"Average Queries Per Second",
"AvgQPS",
"Will calculate the overall average queries Per second. Further on it will save the totaltime of each query, the failure and the success");
this.penalty=penalty;
}



@Override
public void close() {
Expand All @@ -30,32 +40,45 @@ protected void qpsClose(){
Model m = ModelFactory.createDefaultModel();
Map<Object, Number[]> map = new HashMap<Object, Number[]>();
Property property = getMetricProperty();
Property penalziedProp = ResourceFactory.createProperty(COMMON.PROP_BASE_URI+"penalized"+shortName);
for(Properties key : dataContainer.keySet()){
Properties value = dataContainer.get(key);
Double avgQps=0.0;
Double penalizedAvgQps=0.0;
for(Object queryID : value.keySet()){
Object[] resArr = (Object[]) value.get(queryID);
Double qps = (long)resArr[1]*1.0/((double)resArr[0]/1000.0);
map.putIfAbsent(queryID, new Number[]{Double.valueOf(0), Long.valueOf(0)});
Double penalizedQPS = (long)resArr[1]*1.0/((double)resArr[7]/1000.0);
map.putIfAbsent(queryID, new Number[]{Double.valueOf(0), Long.valueOf(0), Double.valueOf(0)});

Number[] current =map.get(queryID);
Long succ = (long)resArr[1]+(Long)current[1];
Double time = (double)resArr[0]+(Double)current[0];
map.put(queryID, new Number[]{time, succ});
Double penTime = (double)resArr[7]+(Double)current[2];
map.put(queryID, new Number[]{time, succ, penTime});
avgQps+=qps;
penalizedAvgQps+=penalizedQPS;
}
avgQps = avgQps/value.size();
penalizedAvgQps = penalizedAvgQps/value.size();
Resource subject = getSubject(key);
m.add(getConnectingStatement(subject));
m.add(subject, property, ResourceFactory.createTypedLiteral(avgQps));
m.add(subject, penalziedProp, ResourceFactory.createTypedLiteral(penalizedAvgQps));

}
Double avgQps=0.0;
Double penalizedAvgQps=0.0;
for(Object queryID : map.keySet()) {
Double qps = (Long)map.get(queryID)[1]*1.0/((Double)map.get(queryID)[0]/1000.0);
Double penalizedQPS = (long)map.get(queryID)[1]*1.0/((double)map.get(queryID)[2]/1000.0);
avgQps+=qps;
penalizedAvgQps+=penalizedQPS;
}
avgQps = avgQps/map.size();
penalizedAvgQps= penalizedAvgQps/map.size();
m.add(getTaskResource(), property, ResourceFactory.createTypedLiteral(avgQps));
m.add(getTaskResource(), penalziedProp, ResourceFactory.createTypedLiteral(penalizedAvgQps));
this.sendData(m);
this.storageManager.commit();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,14 @@ public void addMetaData(Properties p) {
String suiteUrl = getUrlWithResourcePrefix(p, COMMON.SUITE_ID_KEY);
String expUrl = getUrlWithResourcePrefix(p, COMMON.EXPERIMENT_ID_KEY);
String taskUrl = getUrlWithResourcePrefix(p, COMMON.EXPERIMENT_TASK_ID_KEY);

String datasetUrl = getUrlWithResourcePrefix(p, COMMON.DATASET_ID_KEY);
String connUrl = getUrlWithResourcePrefix(p, COMMON.CONNECTION_ID_KEY);
String conName = p.getProperty(COMMON.CONNECTION_ID_KEY);
if(p.containsKey(COMMON.CONNECTION_VERSION_KEY)){
conName+="-"+p.getProperty(COMMON.CONNECTION_VERSION_KEY);
}
String connUrl = getUrlWithResourcePrefix(conName);

String actualTaskID = getUrlWithResourcePrefix(p, COMMON.EXPERIMENT_TASK_CLASS_ID_KEY);


Expand All @@ -77,6 +83,9 @@ public void addMetaData(Properties p) {
addExtraMetadata(p, taskUrl);
metricResults.add(metricResults.createResource(datasetUrl), RDFS.label, p.getProperty(COMMON.DATASET_ID_KEY));
metricResults.add(metricResults.createResource(connUrl), RDFS.label, p.getProperty(COMMON.CONNECTION_ID_KEY));
if(p.containsKey(COMMON.CONNECTION_VERSION_KEY)) {
metricResults.add(metricResults.createResource(connUrl), ResourceFactory.createProperty(getUrlWithPropertyPrefix("version")), p.getProperty(COMMON.CONNECTION_VERSION_KEY));
}

if(p.containsKey(COMMON.QUERY_STATS)) {
Model queryStats = (Model) p.get(COMMON.QUERY_STATS);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ public static Collection<Object[]> data() {
//Test if 2000 will be used instead of provided 1000
testConfigs.add(new Object[] { new QPSMetric(2000), "src/test/resources/nt/qpspenaltytest2.nt", true});
testConfigs.add(new Object[] { new AvgQPSMetric(), "src/test/resources/nt/avgqpstest.nt", false});
testConfigs.add(new Object[] { new AvgQPSMetric(2000), "src/test/resources/nt/penaltyavgqpstest.nt", true});

testConfigs.add(new Object[] { new NoQMetric(), "src/test/resources/nt/noqtest.nt", false});
testConfigs.add(new Object[] { new QMPHMetric(), "src/test/resources/nt/qmphtest.nt", false});
testConfigs.add(new Object[] { new EachQueryMetric(), "src/test/resources/nt/eqtest.nt", false});
Expand Down
5 changes: 5 additions & 0 deletions iguana.resultprocessor/src/test/resources/nt/avgqpstest.nt
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,23 @@
<http://iguana-benchmark.eu/resource/1/1/1/0> <http://iguana-benchmark.eu/properties/workerID> "0" .
<http://iguana-benchmark.eu/resource/1/1/1/0> <http://iguana-benchmark.eu/properties/metric> <http://iguana-benchmark.eu/resource/AvgQPS> .
<http://iguana-benchmark.eu/resource/1/1/1/0> <http://iguana-benchmark.eu/properties/AvgQPS> "4.5"^^<http://www.w3.org/2001/XMLSchema#double> .
<http://iguana-benchmark.eu/resource/1/1/1/0> <http://iguana-benchmark.eu/properties/penalizedAvgQPS> "4.5"^^<http://www.w3.org/2001/XMLSchema#double> .
<http://iguana-benchmark.eu/resource/1/1/1/0> <http://iguana-benchmark.eu/properties/noOfQueries> "2"^^<http://www.w3.org/2001/XMLSchema#int> .

<http://iguana-benchmark.eu/resource/1/1/1/1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://iguana-benchmark.eu/class/Worker> .
<http://iguana-benchmark.eu/resource/1/1/1/1> <http://iguana-benchmark.eu/properties/workerID> "1" .
<http://iguana-benchmark.eu/resource/1/1/1/1> <http://iguana-benchmark.eu/properties/metric> <http://iguana-benchmark.eu/resource/AvgQPS> .
<http://iguana-benchmark.eu/resource/1/1/1/1> <http://iguana-benchmark.eu/properties/AvgQPS> "3.3333333333333335"^^<http://www.w3.org/2001/XMLSchema#double> .
<http://iguana-benchmark.eu/resource/1/1/1/1> <http://iguana-benchmark.eu/properties/noOfQueries> "2"^^<http://www.w3.org/2001/XMLSchema#int> .
<http://iguana-benchmark.eu/resource/1/1/1/1> <http://iguana-benchmark.eu/properties/penalizedAvgQPS> "3.3333333333333335"^^<http://www.w3.org/2001/XMLSchema#double> .


<http://iguana-benchmark.eu/resource/1/1/1> <http://iguana-benchmark.eu/properties/workerResult> <http://iguana-benchmark.eu/resource/1/1/1/1> .
<http://iguana-benchmark.eu/resource/1/1/1> <http://iguana-benchmark.eu/properties/metric> <http://iguana-benchmark.eu/resource/AvgQPS> .
<http://iguana-benchmark.eu/resource/1/1/1> <http://iguana-benchmark.eu/properties/workerResult> <http://iguana-benchmark.eu/resource/1/1/1/0> .
<http://iguana-benchmark.eu/resource/1/1/1> <http://iguana-benchmark.eu/properties/AvgQPS> "4.285714285714286"^^<http://www.w3.org/2001/XMLSchema#double> .
<http://iguana-benchmark.eu/resource/1/1/1> <http://iguana-benchmark.eu/properties/penalizedAvgQPS> "4.285714285714286"^^<http://www.w3.org/2001/XMLSchema#double> .


<http://iguana-benchmark.eu/resource/AvgQPS> <http://www.w3.org/2000/01/rdf-schema#label> "Average Queries Per Second" .
<http://iguana-benchmark.eu/resource/AvgQPS> <http://www.w3.org/2000/01/rdf-schema#comment> "Will calculate the overall average queries Per second. Further on it will save the totaltime of each query, the failure and the success" .
Expand Down
27 changes: 27 additions & 0 deletions iguana.resultprocessor/src/test/resources/nt/penaltyavgqpstest.nt
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
<http://iguana-benchmark.eu/resource/1/1/1/0> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://iguana-benchmark.eu/class/Worker> .
<http://iguana-benchmark.eu/resource/1/1/1/0> <http://iguana-benchmark.eu/properties/workerID> "0" .
<http://iguana-benchmark.eu/resource/1/1/1/0> <http://iguana-benchmark.eu/properties/metric> <http://iguana-benchmark.eu/resource/AvgQPS> .
<http://iguana-benchmark.eu/resource/1/1/1/0> <http://iguana-benchmark.eu/properties/AvgQPS> "4.5"^^<http://www.w3.org/2001/XMLSchema#double> .
<http://iguana-benchmark.eu/resource/1/1/1/0> <http://iguana-benchmark.eu/properties/penalizedAvgQPS> "4.5"^^<http://www.w3.org/2001/XMLSchema#double> .
<http://iguana-benchmark.eu/resource/1/1/1/0> <http://iguana-benchmark.eu/properties/noOfQueries> "2"^^<http://www.w3.org/2001/XMLSchema#int> .

<http://iguana-benchmark.eu/resource/1/1/1/1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://iguana-benchmark.eu/class/Worker> .
<http://iguana-benchmark.eu/resource/1/1/1/1> <http://iguana-benchmark.eu/properties/workerID> "1" .
<http://iguana-benchmark.eu/resource/1/1/1/1> <http://iguana-benchmark.eu/properties/metric> <http://iguana-benchmark.eu/resource/AvgQPS> .
<http://iguana-benchmark.eu/resource/1/1/1/1> <http://iguana-benchmark.eu/properties/AvgQPS> "3.3333333333333335"^^<http://www.w3.org/2001/XMLSchema#double> .
<http://iguana-benchmark.eu/resource/1/1/1/1> <http://iguana-benchmark.eu/properties/noOfQueries> "2"^^<http://www.w3.org/2001/XMLSchema#int> .
<http://iguana-benchmark.eu/resource/1/1/1/1> <http://iguana-benchmark.eu/properties/penalizedAvgQPS> "3.3333333333333335"^^<http://www.w3.org/2001/XMLSchema#double> .


<http://iguana-benchmark.eu/resource/1/1/1> <http://iguana-benchmark.eu/properties/workerResult> <http://iguana-benchmark.eu/resource/1/1/1/1> .
<http://iguana-benchmark.eu/resource/1/1/1> <http://iguana-benchmark.eu/properties/metric> <http://iguana-benchmark.eu/resource/AvgQPS> .
<http://iguana-benchmark.eu/resource/1/1/1> <http://iguana-benchmark.eu/properties/workerResult> <http://iguana-benchmark.eu/resource/1/1/1/0> .
<http://iguana-benchmark.eu/resource/1/1/1> <http://iguana-benchmark.eu/properties/AvgQPS> "4.285714285714286"^^<http://www.w3.org/2001/XMLSchema#double> .
<http://iguana-benchmark.eu/resource/1/1/1> <http://iguana-benchmark.eu/properties/penalizedAvgQPS> "3.0793650793650795"^^<http://www.w3.org/2001/XMLSchema#double> .


<http://iguana-benchmark.eu/resource/AvgQPS> <http://www.w3.org/2000/01/rdf-schema#label> "Average Queries Per Second" .
<http://iguana-benchmark.eu/resource/AvgQPS> <http://www.w3.org/2000/01/rdf-schema#comment> "Will calculate the overall average queries Per second. Further on it will save the totaltime of each query, the failure and the success" .
<http://iguana-benchmark.eu/resource/AvgQPS> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://iguana-benchmark.eu/class/Metric> .
<http://iguana-benchmark.eu/resource/AvgQPS> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://iguana-benchmark.eu/class/metric/AvgQPS> .
<http://iguana-benchmark.eu/resource/AvgQPS> <http://www.w3.org/2000/01/rdf-schema#label> "AvgQPS" .
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.aksw</groupId>
<artifactId>iguana-parent</artifactId>
<version>3.2.1</version>
<version>3.3.0</version>
<name>Iguana Parent </name>
<licenses>
<license>
Expand Down
Binary file modified schema/iguana-result-schema.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
5 changes: 4 additions & 1 deletion schema/iguana-schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@
"endpoint": { "type": "string" },
"updateEndpoint": { "type": "string" },
"user": { "type": "string" },
"password": { "type": "string" }
"password": { "type": "string" },
"version": {
"type": "string"
}
},
"required": ["endpoint"]
},
Expand Down
20 changes: 17 additions & 3 deletions schema/iguana.owl
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@

<owl:Ontology rdf:about="https://iguana-benchmark.eu/ontology/3.0.0/iguana.owl">
<rdfs:label xml:lang="en">Iguana results ontology</rdfs:label>
<owl:versionInfo>3.0.0</owl:versionInfo>
<owl:versionInfo>3.3.0</owl:versionInfo>
<dcterms:created>2020/09/18</dcterms:created>
<dcterms:lastModified>2020/09/18</dcterms:lastModified>
<dc:title xml:lang="en">Iguana results ontology</dc:title>
Expand Down Expand Up @@ -111,7 +111,7 @@

<owl:Class rdf:about="&iont;metric/AvgQPS">
<rdfs:label>Average QPS Metric</rdfs:label>
<rdfs:comment xml:lang="en">Average Queries Per Second Metric. Annotates a Task or Worker if they use this metric.</rdfs:comment>
<rdfs:comment xml:lang="en">Average Queries Per Second Metric. Annotates a Task or Worker if they use this metric.</rdfs:comment>
<rdfs:subClassOf rdf:resource="&iont;Metric" />
</owl:Class>

Expand All @@ -130,7 +130,7 @@
<rdfs:domain rdf:resource="&iont;Task"/>
<rdfs:range rdf:resource="&iont;Connection"/>
</owl:ObjectProperty>

<owl:ObjectProperty rdf:about="&iprop;dataset">
<rdfs:label>dataset</rdfs:label>
<rdfs:comment xml:lang="en">Assigns a Dataset to a Task.</rdfs:comment>
Expand Down Expand Up @@ -186,6 +186,12 @@

<!-- DatatypeProperty definitions -->

<owl:DatatypeProperty rdf:about="&iprop;version">
<rdfs:label>version</rdfs:label>
<rdfs:comment xml:lang="en">Version of the triple store tested.</rdfs:comment>
<rdfs:domain rdf:resource="&iont;Connection"/>
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>

<owl:DatatypeProperty rdf:about="&iprop;timeLimit">
<rdfs:label>timeLimit</rdfs:label>
Expand Down Expand Up @@ -400,6 +406,14 @@
<rdfs:domain rdf:resource="&iont;Task"/>
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#double"/>
</owl:DatatypeProperty>

<owl:DatatypeProperty rdf:about="&iprop;penalizedAvgQPS">
<rdfs:label>penalizedAvgQPS</rdfs:label>
<rdfs:comment xml:lang="en">The average number of queries answered successfully per second value using the penaltyQPS.</rdfs:comment>
<rdfs:domain rdf:resource="&iont;Worker"/>
<rdfs:domain rdf:resource="&iont;Task"/>
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#double"/>
</owl:DatatypeProperty>


<owl:DatatypeProperty rdf:about="&iprop;NoQ">
Expand Down

0 comments on commit f0953c7

Please sign in to comment.