Experiment with metrics and prometheus exporter with otel4s

build.sbt

@@ -8,7 +8,8 @@ inThisBuild(
     organization  := "org.lichess.search",
     run / fork    := true,
     run / javaOptions += "-Dconfig.override_with_env_vars=true",
-    semanticdbEnabled       := true, // for scalafix
+    semanticdbEnabled := true, // for scalafix
+    resolvers ++= ourResolvers,
     Compile / doc / sources := Seq.empty,
     publishTo               := Option(Resolver.file("file", new File(sys.props.getOrElse("publishTo", ""))))
   )
@@ -47,7 +48,8 @@ lazy val elastic = project
       catsCore,
       catsEffect,
       elastic4sJavaClient,
-      elastic4sCatsEffect
+      elastic4sCatsEffect,
+      otel4sCore
     )
   )
   .dependsOn(core)
@@ -71,7 +73,6 @@ lazy val ingestor = (project in file("modules/ingestor"))
     commonSettings,
     publish        := {},
     publish / skip := true,
-    resolvers += lilaMaven,
     libraryDependencies ++= Seq(
       chess,
       catsCore,
@@ -86,10 +87,14 @@ lazy val ingestor = (project in file("modules/ingestor"))
       jsoniterCore,
       jsoniterMacro,
       circe,
+      http4sServer,
       mongo4catsCore,
       mongo4catsCirce,
       log4Cats,
       logback,
+      otel4sMetricts,
+      otel4sSdk,
+      otel4sPrometheusExporter,
       weaver,
       weaverScalaCheck,
       testContainers
@@ -132,6 +137,9 @@ lazy val app = (project in file("modules/app"))
       cirisHtt4s,
       log4Cats,
       logback,
+      otel4sMetricts,
+      otel4sSdk,
+      otel4sPrometheusExporter,
       weaver,
       testContainers
     ),

modules/app/src/main/scala/app.resources.scala

@@ -2,10 +2,11 @@ package lila.search
 package app
 
 import cats.effect.*
+import org.typelevel.otel4s.metrics.Meter
 
 class AppResources(val esClient: ESClient[IO])
 
 object AppResources:
 
-  def instance(conf: AppConfig): Resource[IO, AppResources] =
+  def instance(conf: AppConfig)(using Meter[IO]): Resource[IO, AppResources] =
     ESClient(conf.elastic.uri).map(AppResources.apply)

modules/app/src/main/scala/app.scala

@@ -2,8 +2,15 @@ package lila.search
 package app
 
 import cats.effect.*
+import cats.syntax.all.*
 import org.typelevel.log4cats.slf4j.{ Slf4jFactory, Slf4jLogger }
 import org.typelevel.log4cats.{ Logger, LoggerFactory }
+import org.typelevel.otel4s.experimental.metrics.*
+import org.typelevel.otel4s.metrics.Meter
+import org.typelevel.otel4s.sdk.*
+import org.typelevel.otel4s.sdk.exporter.prometheus.PrometheusMetricExporter
+import org.typelevel.otel4s.sdk.metrics.SdkMetrics
+import org.typelevel.otel4s.sdk.metrics.exporter.MetricExporter
 
 object App extends IOApp.Simple:
 
@@ -14,16 +21,23 @@ object App extends IOApp.Simple:
 
   def app: Resource[IO, Unit] =
     for
-      config <- AppConfig.load.toResource
-      _      <- Logger[IO].info(s"Starting lila-search with config: $config").toResource
-      res    <- AppResources.instance(config)
-      _      <- SearchApp(res, config).run()
+      given MetricExporter.Pull[IO] <- PrometheusMetricExporter.builder[IO].build.toResource
+      given Meter[IO]               <- mkMeter
+      config                        <- AppConfig.load.toResource
+      _   <- Logger[IO].info(s"Starting lila-search with config: $config").toResource
+      _   <- RuntimeMetrics.register[IO]
+      res <- AppResources.instance(config)
+      _   <- mkServer(res, config)
     yield ()
 
-class SearchApp(res: AppResources, config: AppConfig)(using Logger[IO], LoggerFactory[IO]):
-  def run(): Resource[IO, Unit] =
+  def mkMeter(using exporter: MetricExporter.Pull[IO]) = SdkMetrics
+    .autoConfigured[IO](_.addMeterProviderCustomizer((b, _) => b.registerMetricReader(exporter.metricReader)))
+    .evalMap(_.meterProvider.get("lila-search"))
+
+  def mkServer(res: AppResources, config: AppConfig)(using MetricExporter.Pull[IO]): Resource[IO, Unit] =
     for
-      httpRoutes <- Routes(res, config.server)
-      server     <- MkHttpServer.apply.newEmber(config.server, httpRoutes.orNotFound)
-      _ <- Logger[IO].info(s"Starting server on ${config.server.host}:${config.server.port}").toResource
+      apiRoutes <- Routes(res, config.server)
+      httpRoutes = apiRoutes <+> mkPrometheusRoutes
+      server <- MkHttpServer().newEmber(config.server, httpRoutes.orNotFound)
+      _      <- Logger[IO].info(s"Starting server on ${config.server.host}:${config.server.port}").toResource
     yield ()

modules/app/src/main/scala/http.routes.prometheus.scala

@@ -0,0 +1,13 @@
+package lila.search
+package app
+
+import cats.effect.IO
+import org.http4s.HttpRoutes
+import org.http4s.server.Router
+import org.typelevel.otel4s.sdk.exporter.prometheus.*
+import org.typelevel.otel4s.sdk.metrics.exporter.MetricExporter
+
+def mkPrometheusRoutes(using exporter: MetricExporter.Pull[IO]): HttpRoutes[IO] =
+  val writerConfig     = PrometheusWriter.Config.default
+  val prometheusRoutes = PrometheusHttpRoutes.routes[IO](exporter, writerConfig)
+  Router("/metrics" -> prometheusRoutes)

modules/app/src/main/scala/http.routes.scala

@@ -13,9 +13,8 @@ def Routes(resources: AppResources, config: HttpServerConfig)(using
     LoggerFactory[IO]
 ): Resource[IO, HttpRoutes[IO]] =
 
-  val healthServiceImpl: HealthService[IO] = HealthServiceImpl(resources.esClient)
-
-  val searchServiceImpl: SearchService[IO] = SearchServiceImpl(resources.esClient)
+  val healthServiceImpl = HealthServiceImpl(resources.esClient)
+  val searchServiceImpl = SearchServiceImpl(resources.esClient)
 
   val search: Resource[IO, HttpRoutes[IO]] =
     SimpleRestJsonBuilder.routes(searchServiceImpl).resource

modules/app/src/main/scala/http.server.scala

@@ -15,7 +15,7 @@ trait MkHttpServer:
 
 object MkHttpServer:
 
-  def apply(using server: MkHttpServer): MkHttpServer = server
+  def apply()(using server: MkHttpServer): MkHttpServer = server
 
   given forAsyncLogger(using Logger[IO]): MkHttpServer = new:
 

modules/app/src/main/scala/service.search.scala

@@ -138,6 +138,12 @@ object SearchServiceImpl:
         case q: Query.Study => q.to[Study].countDef
         case q: Query.Team  => q.to[Team].countDef
 
+    def index(query: Query) = query match
+      case _: Query.Forum => Index.Forum
+      case _: Query.Game  => Index.Game
+      case _: Query.Study => Index.Study
+      case _: Query.Team  => Index.Team
+
   import smithy4s.json.Json.given
   import com.github.plokhotnyuk.jsoniter_scala.core.*
 

modules/app/src/test/scala/IntegrationSuite.scala

@@ -9,6 +9,9 @@ import lila.search.spec.*
 import org.http4s.Uri
 import org.typelevel.log4cats.noop.{ NoOpFactory, NoOpLogger }
 import org.typelevel.log4cats.{ Logger, LoggerFactory }
+import org.typelevel.otel4s.metrics.Meter
+import org.typelevel.otel4s.sdk.exporter.prometheus.PrometheusMetricExporter
+import org.typelevel.otel4s.sdk.metrics.exporter.MetricExporter
 import smithy4s.Timestamp
 import weaver.*
 
@@ -18,6 +21,7 @@ object IntegrationSuite extends IOSuite:
 
   given Logger[IO]        = NoOpLogger[IO]
   given LoggerFactory[IO] = NoOpFactory[IO]
+  given Meter[IO]         = Meter.noop[IO]
 
   private val uri = Uri.unsafeFromString("http://localhost:9999")
 
@@ -29,8 +33,9 @@ object IntegrationSuite extends IOSuite:
     for
       elastic <- ElasticSearchContainer.start
       config = testAppConfig(elastic)
-      res <- AppResources.instance(config)
-      _   <- SearchApp(res, config).run()
+      res                           <- AppResources.instance(config)
+      given MetricExporter.Pull[IO] <- PrometheusMetricExporter.builder[IO].build.toResource
+      _                             <- App.mkServer(res, config)
     yield res
 
   def testAppConfig(elastic: ElasticConfig) = AppConfig(

modules/e2e/src/test/scala/CompatSuite.scala

@@ -5,11 +5,14 @@ import akka.actor.ActorSystem
 import cats.effect.{ IO, Resource }
 import com.comcast.ip4s.*
 import com.sksamuel.elastic4s.Indexable
-import lila.search.app.{ AppConfig, AppResources, ElasticConfig, HttpServerConfig, SearchApp }
+import lila.search.app.{ App, AppConfig, AppResources, ElasticConfig, HttpServerConfig }
 import lila.search.client.{ SearchClient, SearchError }
 import lila.search.spec.{ CountOutput, Query, SearchOutput, Source }
 import org.typelevel.log4cats.noop.{ NoOpFactory, NoOpLogger }
 import org.typelevel.log4cats.{ Logger, LoggerFactory }
+import org.typelevel.otel4s.metrics.Meter
+import org.typelevel.otel4s.sdk.exporter.prometheus.PrometheusMetricExporter
+import org.typelevel.otel4s.sdk.metrics.exporter.MetricExporter
 import play.api.libs.ws.*
 import play.api.libs.ws.ahc.*
 
@@ -20,15 +23,19 @@ object CompatSuite extends weaver.IOSuite:
 
   given Logger[IO]        = NoOpLogger[IO]
   given LoggerFactory[IO] = NoOpFactory[IO]
+  given Meter[IO]         = Meter.noop[IO]
 
   override type Res = SearchClient
 
   override def sharedResource: Resource[IO, Res] =
     val res = AppResources(fakeClient)
-    SearchApp(res, testAppConfig)
-      .run()
-      .flatMap(_ => wsClient)
-      .map(SearchClient.play(_, "http://localhost:9999/api"))
+    for
+      given MetricExporter.Pull[IO] <- PrometheusMetricExporter.builder[IO].build.toResource
+      res <- App
+        .mkServer(res, testAppConfig)
+        .flatMap(_ => wsClient)
+        .map(SearchClient.play(_, "http://localhost:9999/api"))
+    yield res
 
   val from = From(0)
   val size = Size(12)