Skip to content

Commit 515d49e

Browse files
pappistpappist
authored
Merge pull request #16 from /issues/15/print_element_ids
prints element Ids on error
2 parents 8117a0a + 3142607 commit 515d49e

File tree

3 files changed

+40
-16
lines changed

3 files changed

+40
-16
lines changed

‎src/main/kotlin/Crawler.kt

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,6 @@
11
package com.incquerylabs.twc.repo.crawler
22

3-
import com.incquerylabs.twc.repo.crawler.data.BRANCH_ID
4-
import com.incquerylabs.twc.repo.crawler.data.CHUNK_SIZE
5-
import com.incquerylabs.twc.repo.crawler.data.CrawlerConfiguration
6-
import com.incquerylabs.twc.repo.crawler.data.MAX_HTTP_POOL_SIZE
7-
import com.incquerylabs.twc.repo.crawler.data.MainConfiguration
8-
import com.incquerylabs.twc.repo.crawler.data.RESOURCE_ID
9-
import com.incquerylabs.twc.repo.crawler.data.REVISION
10-
import com.incquerylabs.twc.repo.crawler.data.Server
11-
import com.incquerylabs.twc.repo.crawler.data.User
12-
import com.incquerylabs.twc.repo.crawler.data.WORKSPACE_ID
3+
import com.incquerylabs.twc.repo.crawler.data.*
134
import com.incquerylabs.twc.repo.crawler.verticles.MainVerticle
145
import com.incquerylabs.twc.repo.crawler.verticles.RESTVerticle
156
import io.vertx.core.DeploymentOptions
@@ -57,12 +48,14 @@ private fun executeCrawler(commandLine: CommandLine, cli: CLI) {
5748
val serverOpt = commandLine.getOptionValue<String>("server")
5849
val portOpt = commandLine.getOptionValue<String>("port")
5950
val isSslEnabled = commandLine.isFlagEnabled("ssl")
51+
val trustAll = commandLine.isFlagEnabled(TRUST_ALL_SSL_CERT)
6052
val instanceNum = commandLine.getOptionValue<String>("instanceNum").toInt()
6153
val workspaceId = commandLine.getOptionValue<String>("workspaceId")
6254
val resourceId = commandLine.getOptionValue<String>("resourceId")
6355
val branchId = commandLine.getOptionValue<String>("branchId")
6456
val revision = commandLine.getOptionValue<String>("revision")
6557
val chunkSize = commandLine.getOptionValue<String>(CHUNK_SIZE).toInt()
58+
val requestTimeout = commandLine.getOptionValue<String>(REQUEST_TIMEOUT).toLong()
6659
val maxPoolSize = commandLine.getOptionValue<String>(MAX_HTTP_POOL_SIZE).toInt()
6760
val debug = commandLine.isFlagEnabled("debug")
6861
val requestSingleElement = commandLine.isFlagEnabled("requestSingleElement")
@@ -157,8 +150,15 @@ private fun executeCrawler(commandLine: CommandLine, cli: CLI) {
157150
debug,
158151
server,
159152
User(usr, pswd),
160-
WebClientOptions().setSsl(isSslEnabled).setMaxPoolSize(maxPoolSize),
161-
chunkSize
153+
WebClientOptions().setSsl(isSslEnabled).setMaxPoolSize(maxPoolSize)
154+
.also {
155+
if (trustAll) {
156+
it.setTrustAll(true)
157+
it.setVerifyHost(false)
158+
}
159+
},
160+
chunkSize,
161+
requestTimeout
162162
)
163163

164164
val restVerticle = RESTVerticle(configuration)
@@ -201,6 +201,11 @@ private fun defineCommandLineInterface(): CLI {
201201
createOption("port", "P", "Set server port number"),
202202
createOption("ssl", "ssl", "SSL server connection. Default: false")
203203
.setFlag(true),
204+
createOption(
205+
TRUST_ALL_SSL_CERT,
206+
TRUST_ALL_SSL_CERT,
207+
"Trusts all SSL certificates and bypasses host verification. Use with caution at your own risk."
208+
).setFlag(true),
204209
createOption("instanceNum", "I", "Set number of RESTVerticle instances. Default: 4")
205210
.setDefaultValue("4"),
206211
createOption("workspaceId", "W", "Select workspace to crawl"),
@@ -218,7 +223,8 @@ private fun defineCommandLineInterface(): CLI {
218223
)
219224
.setDefaultValue("2000"),
220225
createOption(MAX_HTTP_POOL_SIZE, "MPS", "Number of concurrent requests. Default: 1")
221-
.setDefaultValue("1")
226+
.setDefaultValue("1"),
227+
createOption(REQUEST_TIMEOUT, "t", "Request timeout. Default: 60 sec").setDefaultValue("60")
222228
)
223229
)
224230
}

‎src/main/kotlin/data/Data.kt

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ data class CrawlerConfiguration(
77
val server: Server,
88
val user: User,
99
val webClientOptions: WebClientOptions,
10-
val chunkSize: Int
10+
val chunkSize: Int,
11+
val requestTimeout: Long = 180
1112
)
1213

1314
data class MainConfiguration(
@@ -95,4 +96,6 @@ const val USER = "user_cookie"
9596
const val SESSION = "session_cookie"
9697

9798
const val CHUNK_SIZE = "chunkSize"
99+
const val REQUEST_TIMEOUT = "timeout"
98100
const val MAX_HTTP_POOL_SIZE = "maxHttpPoolSize"
101+
const val TRUST_ALL_SSL_CERT = "trustAll"

‎src/main/kotlin/verticles/RESTVerticle.kt

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ class RESTVerticle(
1616
val serverPath = configuration.server.path
1717
val port = configuration.server.port
1818
val chunkSize = configuration.chunkSize
19+
val timeout = configuration.requestTimeout * 1_000L
1920

2021
override fun start() {
2122
val twcMap = vertx.sharedData().getLocalMap<Any, Any>(TWCMAP)
@@ -110,6 +111,7 @@ class RESTVerticle(
110111
.putHeader("content-type", "text/plain")
111112
.putHeader("Cookie", "${twcMap[USER]}")
112113
.putHeader("Cookie", "${twcMap[SESSION]}")
114+
.timeout(timeout)
113115
.sendBuffer(Buffer.buffer(elementIds.joinToString(","))) { ar ->
114116
if (ar.succeeded()) {
115117
if (ar.result().statusCode() == 200) {
@@ -166,18 +168,24 @@ class RESTVerticle(
166168
}
167169

168170
} else {
169-
println("getElements: ${ar.result().statusCode()} : ${ar.result().statusMessage()}")
171+
println("Error on requesting elements: ${ar.result().statusCode()} : ${ar.result().statusMessage()}")
172+
printElementIds(elementIds)
170173
myError()
171174
}
172175
} else {
173176
println("Query Root Element failed: ${ar.cause().message}")
177+
printElementIds(elementIds)
174178
myError()
175179
}
176180

177181
}
178182

179183
}
180184

185+
private fun printElementIds(elementIds: List<String>) {
186+
println(" Element IDs: ${elementIds.joinToString(",")}")
187+
}
188+
181189
private fun queryPrepared(elementSize: Long) {
182190
vertx.sharedData().getCounter(QUERIES) { res ->
183191
if (res.succeeded()) {
@@ -226,6 +234,7 @@ class RESTVerticle(
226234
.putHeader("content-type", "application/ld+json")
227235
.putHeader("Cookie", "${twcMap[USER]}")
228236
.putHeader("Cookie", "${twcMap[SESSION]}")
237+
.timeout(timeout)
229238
.sendJson(JsonObject()) { ar ->
230239
if (ar.succeeded()) {
231240
if (ar.result().statusCode() == 200) {
@@ -273,6 +282,7 @@ class RESTVerticle(
273282
.putHeader("content-type", "application/ld+json")
274283
.putHeader("Cookie", "${twcMap[USER]}")
275284
.putHeader("Cookie", "${twcMap[SESSION]}")
285+
.timeout(timeout)
276286
.send { ar ->
277287
if (ar.succeeded()) {
278288
if (ar.result().statusCode() == 200) {
@@ -320,6 +330,7 @@ class RESTVerticle(
320330
.putHeader("content-type", "application/ld+json")
321331
.putHeader("Cookie", "${twcMap[USER]}")
322332
.putHeader("Cookie", "${twcMap[SESSION]}")
333+
.timeout(timeout)
323334
.send { ar ->
324335
if (ar.succeeded()) {
325336
if (ar.result().statusCode() == 200) {
@@ -368,6 +379,7 @@ class RESTVerticle(
368379
.putHeader("content-type", "application/ld+json")
369380
.putHeader("Cookie", "${twcMap[USER]}")
370381
.putHeader("Cookie", "${twcMap[SESSION]}")
382+
.timeout(timeout)
371383
.send { ar ->
372384
if (ar.succeeded()) {
373385
if (ar.result().statusCode() == 200) {
@@ -408,6 +420,7 @@ class RESTVerticle(
408420
.putHeader("content-type", "application/ld+json")
409421
.putHeader("Cookie", "${twcMap[USER]}")
410422
.putHeader("Cookie", "${twcMap[SESSION]}")
423+
.timeout(timeout)
411424
.send { ar ->
412425
if (ar.succeeded()) {
413426
if (ar.result().statusCode() == 200) {
@@ -452,6 +465,7 @@ class RESTVerticle(
452465
.putHeader("content-type", "application/json")
453466
.putHeader("Accept", "text/html")
454467
.putHeader("Authorization", "${twcMap["credential"]}")
468+
.timeout(timeout)
455469
.send { ar ->
456470
if (ar.succeeded()) {
457471
if (ar.result().statusCode() == 204) {
@@ -494,6 +508,7 @@ class RESTVerticle(
494508
.putHeader("content-type", "application/ld+json")
495509
.putHeader("Cookie", "${twcMap[USER]}")
496510
.putHeader("Cookie", "${twcMap[SESSION]}")
511+
.timeout(timeout)
497512
.send { ar ->
498513
if (ar.succeeded()) {
499514
if (ar.result().statusCode() == 204) {
@@ -528,7 +543,7 @@ class RESTVerticle(
528543
.putHeader("content-type", "application/ld+json")
529544
.putHeader("Cookie", "${twcMap[USER]}")
530545
.putHeader("Cookie", "${twcMap[SESSION]}")
531-
.timeout(2000)
546+
.timeout(timeout)
532547
.send { ar ->
533548
if (ar.succeeded()) {
534549
if (ar.result().statusCode() == 200) {

0 commit comments

Comments
 (0)