Mock nomatim server using wiremock.

The README explains how to create mock data.

.gitignore

@@ -11,3 +11,4 @@ target
 *.log
 *.old
 tmp/
+wiremock-standalone*.jar

README.md

@@ -140,6 +140,37 @@ Paste (using `:paste`) the following code snippet :
     // then apply the transformers to test    
 ```
 
+## Unit tests with external API
+
+External APIs are mocked using Wiremock. Wiremock has 2 benefits:
+
+- using a JAR, we can capture every call to an API and save the results for further use
+- then from scala tests, we can start a wiremock server and get these results.
+
+### Capture external API 
+
+Download the standalone JAR from `http://repo1.maven.org/maven2/com/github/tomakehurst/wiremock-standalone/2.25.1/wiremock-standalone-2.25.1.jar` and save it into the `src/test/resources/wiremock` folder.
+
+Launch the JAR by replacing `https://external.api.com` with your own API: 
+
+```bash
+java -jar wiremock-standalone-2.25.1.jar --verbose --proxy-all="https://external.api.com" --record-mappings
+```
+
+Then in order to save the API results, change the API url to `http://localhost:8080` within the requests.
+
+For example, to save nominatim results, you can do:
+```bash
+java -jar wiremock-standalone-2.25.1.jar --verbose --proxy-all="http://nominatim.services.arlas.io" --record-mappings
+curl "http://localhost:8080/reverse.php?format=json&lat=41.270568&lon=6.6701225&zoom=10"
+```
+
+The results will be saved into the resources folder, which is used by scala tests.
+
+### Use mock server from scala tests
+
+A test class can extend the trait `ArlasMockServer`, which automatically starts and stops the mock server.
+
 # Contributing
 
 Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details on our code of conduct, and the process for submitting us pull requests.

build.sbt

@@ -17,7 +17,9 @@ val sparkCassandraConnector = "com.datastax.spark" %% "spark-cassandra-connector
 
 val cassandra = Seq(sparkCassandraConnector)
 
-val scalaTest = "org.scalatest" %% "scalatest" % "2.2.5"
+val scalaTest = "org.scalatest" %% "scalatest" % "2.2.5" % Test
+val wiremockStandalone = "com.github.tomakehurst" % "wiremock-standalone" % "2.25.1" % Test
+val tests = Seq(scalaTest, wiremockStandalone)
 
 val elasticSearch = "org.elasticsearch" %% "elasticsearch-spark-20" % "6.4.0" % "provided"
 val elastic = Seq(elasticSearch)
@@ -37,7 +39,7 @@ lazy val arlasProc = (project in file("."))
     libraryDependencies ++= elastic,
     libraryDependencies ++= geotools,
     libraryDependencies ++= arlas,
-    libraryDependencies += scalaTest % Test
+    libraryDependencies ++= tests
 
     )
 

src/test/resources/wiremock/__files/body-reverse.php-HEW8N.json

@@ -0,0 +1 @@
+{"place_id":197984090,"licence":"Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright","osm_type":"relation","osm_id":34917,"lat":"43.6434","lon":"1.37799","display_name":"Blagnac, Toulouse, Haute-Garonne, Occitanie, France métropolitaine, 31700, France","address":{"city":"Blagnac","county":"Toulouse","state":"Occitanie","country":"France","postcode":"31700","country_code":"fr"},"boundingbox":["43.6153211","43.6665816","1.3473655","1.4068502"]}

src/test/resources/wiremock/__files/body-reverse.php-WOqlB.json

@@ -0,0 +1 @@
+{"place_id":199056147,"licence":"Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright","osm_type":"relation","osm_id":5811926,"lat":"44.5818895","lon":"2.37392262700056","display_name":"Conques-en-Rouergue, Rodez, Aveyron, Occitanie, France métropolitaine, 12320, France","address":{"city":"Conques-en-Rouergue","county":"Rodez","state":"Occitanie","country":"France","postcode":"12320","country_code":"fr"},"boundingbox":["44.4985817","44.665169","2.3152872","2.464032"]}

src/test/resources/wiremock/__files/body-reverse.php-dp8tx.json

@@ -0,0 +1 @@
+{"place_id":218390850,"licence":"Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright","osm_type":"relation","osm_id":124909,"lat":"43.6402778","lon":"0.02","display_name":"Tasque, Mirande, Gers, Occitania, Metropolitan France, 32160, France","address":{"city":"Tasque","county":"Mirande","state":"Occitanie","country":"France","postcode":"32160","country_code":"fr"},"boundingbox":["43.6207693","43.6536688","-0.0167693","0.0406909"]}

src/test/resources/wiremock/__files/body-reverse.php-smE8J.json

@@ -0,0 +1 @@
+{"error":"Unable to geocode"}

src/test/resources/wiremock/mappings/mapping-reverse.php-HEW8N.json

@@ -0,0 +1,24 @@
+{
+  "id" : "a819abf5-5d89-3c46-b74f-f71830edaf6d",
+  "request" : {
+    "url" : "/reverse.php?format=json&lat=43.636883&lon=1.373484&zoom=10",
+    "method" : "GET"
+  },
+  "response" : {
+    "status" : 200,
+    "bodyFileName" : "body-reverse.php-HEW8N.json",
+    "headers" : {
+      "Date" : "Wed, 13 Nov 2019 14:59:42 GMT",
+      "Server" : "Apache/2.4.29 (Ubuntu)",
+      "Access-Control-Allow-Origin" : "*",
+      "Access-Control-Allow-Methods" : "OPTIONS,GET",
+      "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload",
+      "Expect-CT" : "max-age=0, report-uri=\"https://openstreetmap.report-uri.com/r/d/ct/reportOnly\"",
+      "Upgrade" : "h2",
+      "Connection" : "Upgrade, close",
+      "Transfer-Encoding" : "chunked",
+      "Content-Type" : "application/json; charset=UTF-8"
+    }
+  },
+  "uuid" : "a819abf5-5d89-3c46-b74f-f71830edaf6d"
+}

src/test/resources/wiremock/mappings/mapping-reverse.php-WOqlB.json

@@ -0,0 +1,24 @@
+{
+  "id" : "18a04d1b-2bda-30b1-8a5b-1640dde422c5",
+  "request" : {
+    "url" : "/reverse.php?format=json&lat=44.636883&lon=2.373484&zoom=10",
+    "method" : "GET"
+  },
+  "response" : {
+    "status" : 200,
+    "bodyFileName" : "body-reverse.php-WOqlB.json",
+    "headers" : {
+      "Date" : "Wed, 13 Nov 2019 14:59:29 GMT",
+      "Server" : "Apache/2.4.29 (Ubuntu)",
+      "Access-Control-Allow-Origin" : "*",
+      "Access-Control-Allow-Methods" : "OPTIONS,GET",
+      "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload",
+      "Expect-CT" : "max-age=0, report-uri=\"https://openstreetmap.report-uri.com/r/d/ct/reportOnly\"",
+      "Upgrade" : "h2",
+      "Connection" : "Upgrade, close",
+      "Transfer-Encoding" : "chunked",
+      "Content-Type" : "application/json; charset=UTF-8"
+    }
+  },
+  "uuid" : "18a04d1b-2bda-30b1-8a5b-1640dde422c5"
+}

src/test/resources/wiremock/mappings/mapping-reverse.php-dp8tx.json

@@ -0,0 +1,21 @@
+{
+  "id" : "46e00f97-abd9-3e9a-b7c6-24cc0234c4ca",
+  "request" : {
+    "url" : "/reverse.php?format=json&lat=43.636883&lon=0.000932&zoom=10",
+    "method" : "GET"
+  },
+  "response" : {
+    "status" : 200,
+    "bodyFileName" : "body-reverse.php-dp8tx.json",
+    "headers" : {
+      "Date" : "Thu, 14 Nov 2019 16:07:27 GMT",
+      "Server" : "Apache/2.4.38 (Ubuntu)",
+      "Access-Control-Allow-Origin" : "*",
+      "Access-Control-Allow-Methods" : "OPTIONS,GET",
+      "Keep-Alive" : "timeout=5, max=100",
+      "Connection" : "Keep-Alive",
+      "Content-Type" : "application/json; charset=UTF-8"
+    }
+  },
+  "uuid" : "46e00f97-abd9-3e9a-b7c6-24cc0234c4ca"
+}

src/test/resources/wiremock/mappings/mapping-reverse.php-smE8J.json

@@ -0,0 +1,24 @@
+{
+  "id" : "ab2b80ed-fe21-3c8a-a440-f985593cb585",
+  "request" : {
+    "url" : "/reverse.php?format=json&lat=41.270568&lon=6.670123&zoom=10",
+    "method" : "GET"
+  },
+  "response" : {
+    "status" : 200,
+    "bodyFileName" : "body-reverse.php-smE8J.json",
+    "headers" : {
+      "Date" : "Wed, 13 Nov 2019 14:59:04 GMT",
+      "Server" : "Apache/2.4.29 (Ubuntu)",
+      "Access-Control-Allow-Origin" : "*",
+      "Access-Control-Allow-Methods" : "OPTIONS,GET",
+      "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload",
+      "Expect-CT" : "max-age=0, report-uri=\"https://openstreetmap.report-uri.com/r/d/ct/reportOnly\"",
+      "Upgrade" : "h2",
+      "Connection" : "Upgrade, close",
+      "Transfer-Encoding" : "chunked",
+      "Content-Type" : "application/json; charset=UTF-8"
+    }
+  },
+  "uuid" : "ab2b80ed-fe21-3c8a-a440-f985593cb585"
+}

src/test/scala/io/arlas/data/transform/ArlasMockServer.scala

@@ -0,0 +1,27 @@
+package io.arlas.data.transform
+
+import com.github.tomakehurst.wiremock.WireMockServer
+import com.github.tomakehurst.wiremock.core.WireMockConfiguration.options
+import io.arlas.data.app.ArlasProcConfig
+import org.scalatest.{BeforeAndAfterEach, Suite}
+
+trait ArlasMockServer extends BeforeAndAfterEach {
+
+  this: Suite =>
+
+  ArlasProcConfig.GEODATA_BASE_PATH = "http://localhost:8080"
+
+  private val wireMockServer = new WireMockServer(
+    options()
+      .port(8080)
+      .usingFilesUnderClasspath("wiremock"))
+
+  override def beforeEach {
+    wireMockServer.start()
+  }
+
+  override def afterEach {
+    wireMockServer.stop()
+  }
+
+}

src/test/scala/io/arlas/data/transform/features/WithGeoDataTest.scala

@@ -1,13 +1,13 @@
 package io.arlas.data.transform.features
 
-import io.arlas.data.transform.ArlasTest
+import io.arlas.data.transform.{ArlasMockServer, ArlasTest}
 import org.apache.spark.sql.types.{DoubleType, StringType}
 import io.arlas.data.sql._
 import org.apache.spark.sql.functions._
 import scala.collection.immutable.ListMap
 import io.arlas.data.transform.ArlasTestHelper._
 
-class WithGeoDataTest extends ArlasTest {
+class WithGeoDataTest extends ArlasTest with ArlasMockServer {
 
   val testSchemaFields = ListMap(
     "id" -> (StringType, true),
@@ -43,7 +43,7 @@ class WithGeoDataTest extends ArlasTest {
             "France",
             "fr",
             "12320"),
-        Seq("id3", 41.270568, 6.6701225, null, null, null, null, null, null)
+        Seq("id3", 41.270568, 6.670123, null, null, null, null, null, null)
       ),
       testSchemaFields
     )