Skip to content
This repository has been archived by the owner on Jul 2, 2020. It is now read-only.

Http request / resonse max size parameter #73

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Http request / resonse max size parameter #73

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fe3c358
implemented configuration parameters for request and response sizes
Apr 5, 2018
2f11c3d
implemented configuration parameters for request and response sizes
Apr 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions src/main/scala/com/twitter/diffy/DiffyServiceModule.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
package com.twitter.diffy

import com.google.inject.Provides
import com.twitter.diffy.analysis.{InMemoryDifferenceCollector, NoiseDifferenceCounter, RawDifferenceCounter, InMemoryDifferenceCounter}
import com.twitter.diffy.proxy.{Target, Settings}
import com.twitter.diffy.analysis.{InMemoryDifferenceCollector, InMemoryDifferenceCounter, NoiseDifferenceCounter, RawDifferenceCounter}
import com.twitter.diffy.proxy.{Settings, Target}
import com.twitter.inject.TwitterModule
import com.twitter.util.TimeConversions._
import com.twitter.util.StorageUnitConversions._
import java.net.InetSocketAddress

import javax.inject.Singleton
import com.twitter.util.Duration
import com.twitter.util.{Duration, StorageUnit}

object DiffyServiceModule extends TwitterModule {
val datacenter =
Expand Down Expand Up @@ -73,6 +75,12 @@ object DiffyServiceModule extends TwitterModule {
var httpsPort =
flag[String]("httpsPort", "443", "Port to be used when using HTTPS as a protocol")

var maxRequestSize =
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These requirements are independent of SSL and are generally useful. Please use these settings in a shared trait between Http and Https. Bonus points for adding this support at a lower finagle layer so that it may be directly configured in DifferenceProxy to support Thirft in addition to http and https.

flag[StorageUnit]("maxRequestSize", 5.megabytes, "Maximum size for HTTPS request, e.g 5.megabytes")

var maxResponseSize =
flag[StorageUnit]("maxResponseSize", 5.megabytes, "Maximum size for HTTPS responses, e.g 5.megabytes")

@Provides
@Singleton
def settings =
Expand All @@ -97,7 +105,9 @@ object DiffyServiceModule extends TwitterModule {
allowHttpSideEffects(),
excludeHttpHeadersComparison(),
skipEmailsWhenNoErrors(),
httpsPort()
httpsPort(),
maxRequestSize(),
maxResponseSize()
)

@Provides
Expand Down
12 changes: 7 additions & 5 deletions src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ package com.twitter.diffy.proxy

import java.net.SocketAddress

import com.twitter.diffy.analysis.{DifferenceAnalyzer, JoinedDifferences, InMemoryDifferenceCollector}
import com.twitter.diffy.analysis.{DifferenceAnalyzer, InMemoryDifferenceCollector, JoinedDifferences}
import com.twitter.diffy.lifter.{HttpLifter, Message}
import com.twitter.diffy.proxy.DifferenceProxy.NoResponseException
import com.twitter.finagle.{Service, Http, Filter}
import com.twitter.finagle.http.{Status, Response, Method, Request}
import com.twitter.util.{Try, Future}
import org.jboss.netty.handler.codec.http.{HttpResponse, HttpRequest}
import com.twitter.finagle.{Filter, Http, Service}
import com.twitter.finagle.http.{Method, Request, Response, Status}
import com.twitter.util.{Future, StorageUnit, Try}
import org.jboss.netty.handler.codec.http.{HttpRequest, HttpResponse}

object HttpDifferenceProxy {
val okResponse = Future.value(Response(Status.Ok))
Expand Down Expand Up @@ -109,6 +109,8 @@ case class SimpleHttpsDifferenceProxy (
override def serviceFactory(serverset: String, label: String) =
HttpService(
Http.client
.withMaxRequestSize( settings.maxRequestSize )
.withMaxResponseSize( settings.maxResponseSize )
.withTls(serverset)
.newService(serverset+":"+settings.httpsPort, label)
)
Expand Down
46 changes: 24 additions & 22 deletions src/main/scala/com/twitter/diffy/proxy/Settings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,29 +2,31 @@ package com.twitter.diffy.proxy

import java.net.InetSocketAddress

import com.twitter.util.Duration
import com.twitter.util.{Duration, StorageUnit}

case class Settings(
datacenter: String,
servicePort:InetSocketAddress,
candidate: Target,
primary: Target,
secondary: Target,
protocol: String,
clientId: String,
pathToThriftJar: String,
serviceClass: String,
serviceName: String,
apiRoot: String,
enableThriftMux: Boolean,
relativeThreshold: Double,
absoluteThreshold: Double,
teamEmail: String,
emailDelay: Duration,
rootUrl: String,
allowHttpSideEffects: Boolean,
excludeHttpHeadersComparison: Boolean,
skipEmailsWhenNoErrors: Boolean,
httpsPort: String)
datacenter: String,
servicePort:InetSocketAddress,
candidate: Target,
primary: Target,
secondary: Target,
protocol: String,
clientId: String,
pathToThriftJar: String,
serviceClass: String,
serviceName: String,
apiRoot: String,
enableThriftMux: Boolean,
relativeThreshold: Double,
absoluteThreshold: Double,
teamEmail: String,
emailDelay: Duration,
rootUrl: String,
allowHttpSideEffects: Boolean,
excludeHttpHeadersComparison: Boolean,
skipEmailsWhenNoErrors: Boolean,
httpsPort: String,
maxRequestSize: StorageUnit,
maxResponseSize: StorageUnit )

case class Target(path: String)
6 changes: 5 additions & 1 deletion src/test/scala/com/twitter/diffy/TestHelper.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import java.net.InetSocketAddress

import com.twitter.diffy.proxy._
import com.twitter.util.TimeConversions._
import com.twitter.util.StorageUnitConversions._
import org.scalatest.mock.MockitoSugar
import com.twitter.diffy.analysis._
import com.twitter.diffy.compare.Difference
import com.twitter.util.StorageUnit

object TestHelper extends MockitoSugar {
lazy val testSettings = Settings(
Expand All @@ -30,7 +32,9 @@ object TestHelper extends MockitoSugar {
allowHttpSideEffects = true,
excludeHttpHeadersComparison = true,
skipEmailsWhenNoErrors = false,
httpsPort = "443"
httpsPort = "443",
maxRequestSize = 5.megabytes,
maxResponseSize = 5.megabytes
)

def makeEmptyJoinedDifferences = {
Expand Down