From 6341401ba816eae573d4a928b83d8b0181e5f695 Mon Sep 17 00:00:00 2001 From: shivam-maharshi Date: Fri, 21 Oct 2016 05:19:06 -0400 Subject: [PATCH] Removed WireMock since it had some issues. Incorporated review feedbacks. --- checkstyle.xml | 4 +- .../yahoo/ycsb/workloads/RestWorkload.java | 48 ++-- rest/pom.xml | 37 +-- .../ycsb/webservice/rest/RestClient.java | 41 +-- .../ycsb/webservice/rest/IntegrationTest.java | 5 +- .../ycsb/webservice/rest/RestClientTest.java | 241 ++++++++++-------- .../webservice/rest/RestTestResource.java | 32 ++- rest/src/test/resources/workload_rest | 10 +- 8 files changed, 224 insertions(+), 194 deletions(-) diff --git a/checkstyle.xml b/checkstyle.xml index 92d1ce37f8..af0065d076 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -144,7 +144,9 @@ LICENSE file. - + + + diff --git a/core/src/main/java/com/yahoo/ycsb/workloads/RestWorkload.java b/core/src/main/java/com/yahoo/ycsb/workloads/RestWorkload.java index f5d9bc65b1..b36c65a99c 100644 --- a/core/src/main/java/com/yahoo/ycsb/workloads/RestWorkload.java +++ b/core/src/main/java/com/yahoo/ycsb/workloads/RestWorkload.java @@ -23,7 +23,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; import com.yahoo.ycsb.ByteIterator; import com.yahoo.ycsb.DB; @@ -165,27 +164,32 @@ public static DiscreteGenerator createOperationGenerator(final Properties p) { private static NumberGenerator getKeyChooser(String requestDistrib, int recordCount, double zipfContant, Properties p) throws WorkloadException { NumberGenerator keychooser = null; - if (requestDistrib.compareTo("exponential") == 0) { - double percentile = Double.parseDouble(p.getProperty(ExponentialGenerator.EXPONENTIAL_PERCENTILE_PROPERTY, - ExponentialGenerator.EXPONENTIAL_PERCENTILE_DEFAULT)); - double frac = Double.parseDouble(p.getProperty(ExponentialGenerator.EXPONENTIAL_FRAC_PROPERTY, - ExponentialGenerator.EXPONENTIAL_FRAC_DEFAULT)); - keychooser = new ExponentialGenerator(percentile, recordCount * frac); - } else if (requestDistrib.compareTo("uniform") == 0) { - keychooser = new UniformIntegerGenerator(0, recordCount - 1); - } else if (requestDistrib.compareTo("zipfian") == 0) { - keychooser = new ZipfianGenerator(recordCount, zipfContant); - } else if (requestDistrib.compareTo("latest") == 0) { - // TODO: To be implemented. - System.err.println("Latest request distribution is not supported"); - } else if (requestDistrib.equals("hotspot")) { - double hotsetfraction = Double + + switch(requestDistrib) { + case "exponential": + double percentile = Double.parseDouble(p.getProperty(ExponentialGenerator.EXPONENTIAL_PERCENTILE_PROPERTY, + ExponentialGenerator.EXPONENTIAL_PERCENTILE_DEFAULT)); + double frac = Double.parseDouble(p.getProperty(ExponentialGenerator.EXPONENTIAL_FRAC_PROPERTY, + ExponentialGenerator.EXPONENTIAL_FRAC_DEFAULT)); + keychooser = new ExponentialGenerator(percentile, recordCount * frac); + break; + case "uniform": + keychooser = new UniformIntegerGenerator(0, recordCount - 1); + break; + case "zipfian": + keychooser = new ZipfianGenerator(recordCount, zipfContant); + break; + case "latest": + throw new WorkloadException("Latest request distribution is not supported for RestWorkload."); + case "hotspot": + double hotsetfraction = Double .parseDouble(p.getProperty(HOTSPOT_DATA_FRACTION, HOTSPOT_DATA_FRACTION_DEFAULT)); - double hotopnfraction = Double + double hotopnfraction = Double .parseDouble(p.getProperty(HOTSPOT_OPN_FRACTION, HOTSPOT_OPN_FRACTION_DEFAULT)); - keychooser = new HotspotIntegerGenerator(0, recordCount - 1, hotsetfraction, hotopnfraction); - } else { - throw new WorkloadException("Unknown request distribution \"" + requestDistrib + "\""); + keychooser = new HotspotIntegerGenerator(0, recordCount - 1, hotsetfraction, hotopnfraction); + break; + default: + throw new WorkloadException("Unknown request distribution \"" + requestDistrib + "\""); } return keychooser; } @@ -209,9 +213,9 @@ protected static NumberGenerator getFieldLengthGenerator(Properties p) throws Wo /** * Reads the trace file and returns a URL map. */ - private static synchronized Map getTrace(String filePath, int recordCount) + private static Map getTrace(String filePath, int recordCount) throws WorkloadException { - Map urlMap = new ConcurrentHashMap(); + Map urlMap = new HashMap(); int count = 0; String line; try { diff --git a/rest/pom.xml b/rest/pom.xml index c5f65c3db5..2fec005943 100644 --- a/rest/pom.xml +++ b/rest/pom.xml @@ -29,8 +29,6 @@ 4.4.4 4.12 1.16.0 - 1.57 - true @@ -60,17 +58,6 @@ com.github.stefanbirkner system-rules ${system-rules.version} - - - com.github.tomakehurst - wiremock - ${wiremock.version} - - - org.mortbay.jetty - servlet-api - - org.glassfish.jersey.core @@ -124,27 +111,5 @@ ${tomcat.version} - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - jar-with-dependencies - - false - - - - package - - single - - - - - - + diff --git a/rest/src/main/java/com/yahoo/ycsb/webservice/rest/RestClient.java b/rest/src/main/java/com/yahoo/ycsb/webservice/rest/RestClient.java index 17fe55b499..bc3d3492b5 100644 --- a/rest/src/main/java/com/yahoo/ycsb/webservice/rest/RestClient.java +++ b/rest/src/main/java/com/yahoo/ycsb/webservice/rest/RestClient.java @@ -28,6 +28,8 @@ import java.util.Vector; import java.util.zip.GZIPInputStream; +import javax.ws.rs.HttpMethod; + import org.apache.http.HttpEntity; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.config.RequestConfig; @@ -53,7 +55,7 @@ * Class responsible for making web service requests for benchmarking purpose. * Using Apache HttpClient over standard Java HTTP API as this is more flexible * and provides better functionality. For example HttpClient can automatically - * handle Redirects and Proxy Authentication which the standard Java API don't. + * handle redirects and proxy authentication which the standard Java API can't. */ public class RestClient extends DB { @@ -64,21 +66,21 @@ public class RestClient extends DB { private static final String LOG_ENABLED = "log.enable"; private static final String HEADERS = "headers"; private static final String COMPRESSED_RESPONSE = "response.compression"; - private static boolean compressedResponse; - private static boolean logEnabled; + private boolean compressedResponse; + private boolean logEnabled; private String urlPrefix; private Properties props; private String[] headers; private CloseableHttpClient client; - private static int conTimeout = 10000; - private static int readTimeout = 10000; - private static int execTimeout = 10000; + private int conTimeout = 10000; + private int readTimeout = 10000; + private int execTimeout = 10000; private volatile Criteria requestTimedout = new Criteria(false); @Override public void init() throws DBException { props = getProperties(); - urlPrefix = props.getProperty(URL_PREFIX, "127.0.0.1:8080"); + urlPrefix = props.getProperty(URL_PREFIX, "http://127.0.0.1:8080"); conTimeout = Integer.valueOf(props.getProperty(CON_TIMEOUT, "10")) * 1000; readTimeout = Integer.valueOf(props.getProperty(READ_TIMEOUT, "10")) * 1000; execTimeout = Integer.valueOf(props.getProperty(EXEC_TIMEOUT, "10")) * 1000; @@ -104,7 +106,7 @@ public Status read(String table, String endpoint, Set fields, HashMap 1.7. - @Before - public void beforeMethod() { - Assume.assumeTrue(Float.parseFloat(System.getProperty("java.specification.version")) > 1.7); - } - // Read success. @Test public void read_200() { - stubFor(get(urlEqualTo("/webService/rest/resource/1")).withHeader("Accept", equalTo("*/*")) - .willReturn(aResponse().withStatus(200).withHeader("Content-Type", "text/xml").withBody(SUCCESS_RESPONSE))); HashMap result = new HashMap(); - Status status = rc.read(null, resource, null, result); + Status status = rc.read(null, VALID_RESOURCE, null, result); assertEquals(Status.OK, status); - assertEquals(result.get(RESPONSE_TAG).toString(), SUCCESS_RESPONSE); + assertEquals(result.get(RESPONSE_TAG).toString(), "HTTP GET response to: "+ VALID_RESOURCE); } - - // Read GZIP response success. Not supported currently. - @Ignore + + // Unauthorized request error. @Test - public void readZip_200() throws DBException { - stubFor(get(urlEqualTo("/webService/rest/resource/1")).withHeader("Accept", equalTo("*/*")).willReturn(aResponse().withStatus(200) - .withHeader("Content-Type", "text/xml").withHeader("Content-Encoding", "gzip").withBody(GZIP_SUCCESS_REPONSE))); + public void read_403() { HashMap result = new HashMap(); - Status status = rc.read(null, resource, null, result); - assertEquals(Status.OK, status); - assertEquals(result.get(RESPONSE_TAG).toString(), SUCCESS_RESPONSE); + Status status = rc.read(null, UNAUTHORIZED_RESOURCE, null, result); + assertEquals(Status.FORBIDDEN, status); } - - // Not found error. + + //Not found error. @Test - public void read_400() { - stubFor(get(urlEqualTo("/webService/rest/resource/1")).withHeader("Accept", equalTo("*/*")) - .willReturn(aResponse().withStatus(404).withHeader("Content-Type", "text/xml").withBody(NOT_FOUND_RESPONSE))); + public void read_404() { HashMap result = new HashMap(); - Status status = rc.read(null, resource, null, result); + Status status = rc.read(null, ABSENT_RESOURCE, null, result); assertEquals(Status.NOT_FOUND, status); - assertEquals(result.get(RESPONSE_TAG).toString(), NOT_FOUND_RESPONSE); } - + + // Server error. @Test - public void read_403() { - stubFor(get(urlEqualTo("/webService/rest/resource/1")).withHeader("Accept", equalTo("*/*")) - .willReturn(aResponse().withStatus(403).withHeader("Content-Type", "text/xml"))); + public void read_500() { HashMap result = new HashMap(); - Status status = rc.read(null, resource, null, result); - assertEquals(Status.FORBIDDEN, status); + Status status = rc.read(null, INVALID_RESOURCE, null, result); + assertEquals(Status.ERROR, status); + } + + //Response delay will cause execution timeout exception. + @Test + public void read_timeout_500() { + HashMap result = new HashMap(); + Status status = rc.read(null, TIMEOUT_RESOURCE, null, result); + assertEquals(Status.ERROR, status); } // Insert success. @Test public void insert_200() { - stubFor(post(urlEqualTo("/webService/rest/resource/1")).withHeader("Accept", equalTo("*/*")).withRequestBody(equalTo(INPUT_DATA)) - .willReturn(aResponse().withStatus(200).withHeader("Content-Type", "text/xml").withBody(SUCCESS_RESPONSE))); HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); - Status status = rc.insert(null, resource, data); + Status status = rc.insert(null, VALID_RESOURCE, data); assertEquals(Status.OK, status); } - - // Response delay will cause execution timeout exception. + + @Test + public void insert_403() { + HashMap data = new HashMap(); + data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); + Status status = rc.insert(null, UNAUTHORIZED_RESOURCE, data); + assertEquals(Status.FORBIDDEN, status); + } + + @Test + public void insert_404() { + HashMap data = new HashMap(); + data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); + Status status = rc.insert(null, ABSENT_RESOURCE, data); + assertEquals(Status.NOT_FOUND, status); + } + @Test public void insert_500() { - stubFor(post(urlEqualTo("/webService/rest/resource/1")).withHeader("Accept", equalTo("*/*")).withRequestBody(equalTo(INPUT_DATA)) - .willReturn(aResponse().withStatus(500).withHeader("Content-Type", "text/xml").withFixedDelay(RESPONSE_DELAY))); HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); - Status status = rc.insert(null, resource, data); + Status status = rc.insert(null, INVALID_RESOURCE, data); + assertEquals(Status.ERROR, status); + } + + @Test + public void insert_timeout_500() { + HashMap data = new HashMap(); + data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); + Status status = rc.insert(null, TIMEOUT_RESOURCE, data); assertEquals(Status.ERROR, status); } // Delete success. @Test public void delete_200() { - stubFor(delete(urlEqualTo("/webService/rest/resource/1")).withHeader("Accept", equalTo("*/*")) - .willReturn(aResponse().withStatus(200).withHeader("Content-Type", "text/xml"))); - Status status = rc.delete(null, resource); + Status status = rc.delete(null, VALID_RESOURCE); assertEquals(Status.OK, status); } + + @Test + public void delete_403() { + Status status = rc.delete(null, UNAUTHORIZED_RESOURCE); + assertEquals(Status.FORBIDDEN, status); + } + + @Test + public void delete_404() { + Status status = rc.delete(null, ABSENT_RESOURCE); + assertEquals(Status.NOT_FOUND, status); + } @Test public void delete_500() { - stubFor(delete(urlEqualTo("/webService/rest/resource/1")).withHeader("Accept", equalTo("*/*")) - .willReturn(aResponse().withStatus(500).withHeader("Content-Type", "text/xml"))); - Status status = rc.delete(null, resource); + Status status = rc.delete(null, INVALID_RESOURCE); + assertEquals(Status.ERROR, status); + } + + @Test + public void delete_timeout_500() { + Status status = rc.delete(null, TIMEOUT_RESOURCE); assertEquals(Status.ERROR, status); } @Test public void update_200() { - stubFor(put(urlEqualTo("/webService/rest/resource/1")).withHeader("Accept", equalTo("*/*")).withRequestBody(equalTo(INPUT_DATA)) - .willReturn(aResponse().withStatus(200).withHeader("Content-Type", "text/xml").withBody(SUCCESS_RESPONSE))); HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); - Status status = rc.update(null, resource, data); + Status status = rc.update(null, VALID_RESOURCE, data); assertEquals(Status.OK, status); } - // Response delay will cause execution timeout exception. + @Test + public void update_403() { + HashMap data = new HashMap(); + data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); + Status status = rc.update(null, UNAUTHORIZED_RESOURCE, data); + assertEquals(Status.FORBIDDEN, status); + } + + @Test + public void update_404() { + HashMap data = new HashMap(); + data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); + Status status = rc.update(null, ABSENT_RESOURCE, data); + assertEquals(Status.NOT_FOUND, status); + } + @Test public void update_500() { - stubFor(put(urlEqualTo("/webService/rest/resource/1")).withHeader("Accept", equalTo("*/*")).withRequestBody(equalTo(INPUT_DATA)) - .willReturn(aResponse().withStatus(500).withHeader("Content-Type", "text/xml"))); HashMap data = new HashMap(); data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); - Status status = rc.update(null, resource, data); + Status status = rc.update(null, INVALID_RESOURCE, data); + assertEquals(Status.ERROR, status); + } + + @Test + public void update_timeout_500() { + HashMap data = new HashMap(); + data.put(DATA_TAG, new StringByteIterator(INPUT_DATA)); + Status status = rc.update(null, TIMEOUT_RESOURCE, data); assertEquals(Status.ERROR, status); } @@ -217,4 +250,8 @@ public void scan() { assertEquals(Status.NOT_IMPLEMENTED, rc.scan(null, null, 0, null, null)); } + private static ServletContainer resourceConfig() { + return new ServletContainer(new ResourceConfig(new ResourceLoader().getClasses())); + } + } diff --git a/rest/src/test/java/com/yahoo/ycsb/webservice/rest/RestTestResource.java b/rest/src/test/java/com/yahoo/ycsb/webservice/rest/RestTestResource.java index ec710f45cb..091ae132b2 100644 --- a/rest/src/test/java/com/yahoo/ycsb/webservice/rest/RestTestResource.java +++ b/rest/src/test/java/com/yahoo/ycsb/webservice/rest/RestTestResource.java @@ -19,6 +19,7 @@ import javax.ws.rs.DELETE; import javax.ws.rs.GET; +import javax.ws.rs.HttpMethod; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; @@ -37,32 +38,41 @@ public class RestTestResource { @GET @Produces(MediaType.TEXT_PLAIN) public Response respondToGET(@PathParam("id") String id) { - if (id.equals("resource_invalid")) - return Response.serverError().build(); - return Response.ok("A dummy response to HTTP GET!").build(); + return processRequests(id, HttpMethod.GET); } - + @POST @Produces(MediaType.TEXT_PLAIN) public Response respondToPOST(@PathParam("id") String id) { - if (id.equals("resource_invalid")) - return Response.serverError().build(); - return Response.ok("A dummy response to HTTP POST!").build(); + return processRequests(id, HttpMethod.POST); } @DELETE @Produces(MediaType.TEXT_PLAIN) public Response respondToDELETE(@PathParam("id") String id) { - if (id.equals("resource_invalid")) - return Response.serverError().build(); - return Response.ok("A dummy response to HTTP DELETE!").build(); + return processRequests(id, HttpMethod.DELETE); } @PUT @Produces(MediaType.TEXT_PLAIN) public Response respondToPUT(@PathParam("id") String id) { + return processRequests(id, HttpMethod.PUT); + } + + private static Response processRequests(String id, String method) { if (id.equals("resource_invalid")) return Response.serverError().build(); - return Response.ok("A dummy response to HTTP PUT!").build(); + else if (id.equals("resource_absent")) + return Response.status(Response.Status.NOT_FOUND).build(); + else if (id.equals("resource_unauthorized")) + return Response.status(Response.Status.FORBIDDEN).build(); + else if (id.equals("resource_timeout")) { + try { + Thread.sleep(6000); + } catch (InterruptedException e) { + // Do nothing. + } + } + return Response.ok("HTTP " + method + " response to: " + id).build(); } } \ No newline at end of file diff --git a/rest/src/test/resources/workload_rest b/rest/src/test/resources/workload_rest index 369c47db20..f51bfa63ae 100644 --- a/rest/src/test/resources/workload_rest +++ b/rest/src/test/resources/workload_rest @@ -42,16 +42,16 @@ operationcount=1 maxexecutiontime=720 # Custom Properties -url.prefix=http://127.0.0.1:8080/webService/rest/resource/ +url.prefix=http://127.0.0.1:PORT/webService/rest/resource/ url.trace.read=/src/test/resource/trace.txt url.trace.insert=/src/test/resource/trace.txt url.trace.update=/src/test/resource/trace.txt url.trace.delete=/src/test/resource/trace.txt # Header must be separated by space. Other delimiters might occur as header values and hence can not be used. -headers=Accept */* Accept-Language en-US,en;q=0.5 Content-Type application/x-www-form-urlencoded user-agent Mozilla/5.0 -timeout.con=10 -timeout.read=10 -timeout.exec=10 +headers=Accept */* Accept-Language en-US,en;q=0.5 Content-Type application/x-www-form-urlencoded user-agent Mozilla/5.0 Connection close +timeout.con=5 +timeout.read=5 +timeout.exec=5 log.enable=false readrecordcount=10000 insertrecordcount=5000