Dependencies: Upgrade Scala to 3.3.3 (alexklibisz#653)

.scalafmt.conf

@@ -1,3 +1,3 @@
 version = "3.8.0"
 maxColumn = 140
-runner.dialect = scala213
+runner.dialect = scala3

build.sbt

@@ -1,7 +1,9 @@
 import ElasticsearchPluginPlugin.autoImport.*
 import org.typelevel.scalacoptions.*
 
-Global / scalaVersion := "2.13.13"
+Global / scalaVersion := "3.3.3"
+
+Global / scalacOptions += "-explain"
 
 lazy val CirceVersion = "0.14.3"
 lazy val ElasticsearchVersion = "8.12.2"
@@ -16,18 +18,6 @@ lazy val TestSettings = Seq(
   libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.18" % Test
 )
 
-lazy val TpolecatSettings = Seq(
-  // Disabling a couple of warnings for test files.
-  // Ideally we would scope this to Test / scalacOptions,
-  // but then compilation fails within IntelliJ.
-  // https://github.com/typelevel/sbt-tpolecat/issues/134
-  scalacOptions ++= Seq(
-    "-Wconf:any:wv",
-    "-Wconf:cat=other-pure-statement:src=test/.*.scala:silent",
-    "-Wconf:cat=w-flag-numeric-widen:src=test/.*.scala:silent"
-  )
-)
-
 lazy val `elastiknn-root` = project
   .in(file("."))
   .settings(
@@ -52,7 +42,6 @@ lazy val `elastiknn-api4s` = project
       "org.elasticsearch" % "elasticsearch-x-content" % ElasticsearchVersion,
       "io.circe" %% "circe-parser" % CirceVersion % Test
     ),
-    TpolecatSettings,
     TestSettings
   )
 
@@ -65,7 +54,6 @@ lazy val `elastiknn-client-elastic4s` = project
     libraryDependencies ++= Seq(
       "com.sksamuel.elastic4s" %% "elastic4s-client-esjava" % Elastic4sVersion
     ),
-    TpolecatSettings,
     TestSettings
   )
 
@@ -79,7 +67,6 @@ lazy val `elastiknn-lucene` = project
       "org.apache.lucene" % "lucene-core" % LuceneVersion,
       "org.apache.lucene" % "lucene-analysis-common" % LuceneVersion % Test
     ),
-    TpolecatSettings,
     TestSettings
   )
 
@@ -97,7 +84,6 @@ lazy val `elastiknn-models` = project
       "--add-exports",
       "java.base/jdk.internal.vm.annotation=ALL-UNNAMED"
     ),
-    TpolecatSettings,
     TestSettings
   )
 
@@ -107,7 +93,6 @@ lazy val `elastiknn-jmh-benchmarks` = project
   .enablePlugins(JmhPlugin)
   .settings(
     Jmh / javaOptions ++= Seq("--add-modules", "jdk.incubator.vector"),
-    TpolecatSettings,
     libraryDependencies ++= Seq(
       "org.eclipse.collections" % "eclipse-collections" % "11.1.0",
       "org.eclipse.collections" % "eclipse-collections-api" % "11.1.0"
@@ -137,18 +122,15 @@ lazy val `elastiknn-plugin` = project
       "com.google.guava" % "failureaccess" % "1.0.2",
       "org.scalanlp" %% "breeze" % "2.1.0" % Test,
       "io.circe" %% "circe-parser" % CirceVersion % Test,
-      "io.circe" %% "circe-generic-extras" % CirceVersion % Test,
-      "ch.qos.logback" % "logback-classic" % "1.5.3" % Test,
-      "com.klibisz.futil" %% "futil" % "0.1.2" % Test
+      "ch.qos.logback" % "logback-classic" % "1.5.3" % Test
     ),
-    TpolecatSettings,
     TestSettings
   )
 
 lazy val `elastiknn-plugin-integration-tests` = project
   .in(file("elastiknn-plugin-integration-tests"))
   .dependsOn(`elastiknn-plugin` % "test->test")
   .settings(
-    TpolecatSettings,
+    libraryDependencies += "io.circe" %% "circe-generic" % CirceVersion % Test,
     TestSettings
   )

docs/pages/performance/fashion-mnist/results.md

@@ -1,10 +1,10 @@
 |Model|Parameters|Recall|Queries per Second|
 |---|---|---|---|
-|eknn-l2lsh|L=100 k=4 w=1024 candidates=500 probes=0|0.378|381.926|
-|eknn-l2lsh|L=100 k=4 w=1024 candidates=1000 probes=0|0.447|315.984|
-|eknn-l2lsh|L=100 k=4 w=1024 candidates=500 probes=3|0.635|298.115|
-|eknn-l2lsh|L=100 k=4 w=1024 candidates=1000 probes=3|0.716|258.478|
-|eknn-l2lsh|L=100 k=4 w=2048 candidates=500 probes=0|0.767|335.131|
-|eknn-l2lsh|L=100 k=4 w=2048 candidates=1000 probes=0|0.846|282.080|
-|eknn-l2lsh|L=100 k=4 w=2048 candidates=500 probes=3|0.921|222.554|
-|eknn-l2lsh|L=100 k=4 w=2048 candidates=1000 probes=3|0.960|202.313|
+|eknn-l2lsh|L=100 k=4 w=1024 candidates=500 probes=0|0.378|382.992|
+|eknn-l2lsh|L=100 k=4 w=1024 candidates=1000 probes=0|0.446|322.386|
+|eknn-l2lsh|L=100 k=4 w=1024 candidates=500 probes=3|0.634|301.434|
+|eknn-l2lsh|L=100 k=4 w=1024 candidates=1000 probes=3|0.716|266.643|
+|eknn-l2lsh|L=100 k=4 w=2048 candidates=500 probes=0|0.767|340.303|
+|eknn-l2lsh|L=100 k=4 w=2048 candidates=1000 probes=0|0.847|294.373|
+|eknn-l2lsh|L=100 k=4 w=2048 candidates=500 probes=3|0.922|233.123|
+|eknn-l2lsh|L=100 k=4 w=2048 candidates=1000 probes=3|0.960|207.765|

elastiknn-api4s/src/main/scala/com/klibisz/elastiknn/api/XContentCodec.scala

@@ -366,6 +366,13 @@ object XContentCodec {
       override def compare(x: Token, y: Token): Int = x.name().compareTo(y.name())
     }
 
+    @inline
+    private def nextToken(p: XContentParser): Unit = {
+      p.nextToken()
+      // Need to return unit.
+      ()
+    }
+
     private def unexpectedValue(text: String, expected: SortedSet[String]): String =
       s"Expected token to be one of [${expected.mkString(",")}] but found [$text]"
 
@@ -475,7 +482,7 @@ object XContentCodec {
                 case n @ Names.VALUES =>
                   assertToken(n, p.nextToken(), START_ARRAY)
                   values = Some(parseFloatArray(p))
-                case _ => p.nextToken()
+                case _ => nextToken(p)
               }
             }
           case START_ARRAY =>
@@ -577,10 +584,10 @@ object XContentCodec {
                   assertToken(n, p.nextToken(), VALUE_BOOLEAN)
                   repeating = Some(p.booleanValue())
                 case Names.SIMILARITY => similarity = Some(Decoder.similarity.decodeUnsafe(p))
-                case _                => p.nextToken()
+                case _                => nextToken(p)
               }
             }
-          case _ => p.nextToken()
+          case _ => nextToken(p)
         }
       }
       (typ, model, dims, similarity, l, k, w, repeating) match {
@@ -630,7 +637,7 @@ object XContentCodec {
               probes = Some(p.intValue())
             case Names.SIMILARITY => similarity = Some(decodeUnsafe[Similarity](p))
             case Names.VEC        => vec = Some(decodeUnsafe[Vec](p))
-            case _                => p.nextToken()
+            case _                => nextToken(p)
           }
         }
         (candidates, field, model, probes, similarity, vec) match {

elastiknn-client-elastic4s/src/main/scala/com/klibisz/elastiknn/client/ElastiknnClient.scala

@@ -1,5 +1,6 @@
 package com.klibisz.elastiknn.client
 
+import com.fasterxml.jackson.module.scala.JavaTypeable
 import com.klibisz.elastiknn.api._
 import com.sksamuel.elastic4s.ElasticDsl._
 import com.sksamuel.elastic4s._
@@ -22,11 +23,11 @@ trait ElastiknnClient[F[_]] extends AutoCloseable {
 
   /** Abstract method for executing a request.
     */
-  def execute[T, U](request: T)(implicit handler: Handler[T, U], manifest: Manifest[U]): F[Response[U]]
+  def execute[T, U](request: T)(implicit handler: Handler[T, U], javaTypeable: JavaTypeable[U]): F[Response[U]]
 
   /** Execute the given request.
     */
-  final def apply[T, U](request: T)(implicit handler: Handler[T, U], manifest: Manifest[U]): F[Response[U]] = execute(request)
+  final def apply[T, U](request: T)(implicit handler: Handler[T, U], javaTypeable: JavaTypeable[U]): F[Response[U]] = execute(request)
 
   /** See ElastiknnRequests.putMapping().
     */
@@ -82,7 +83,7 @@ trait ElastiknnClient[F[_]] extends AutoCloseable {
       override def build(t: SearchRequest): ElasticRequest = SearchHandler.build(t)
       override def responseHandler: ResponseHandler[SearchResponse] = (response: HttpResponse) => {
         val handled: Either[ElasticError, SearchResponse] = SearchHandler.responseHandler.handle(response)
-        handled.map { sr: SearchResponse =>
+        handled.map { (sr: SearchResponse) =>
           val hitsWithIds = sr.hits.hits.map(h =>
             h.copy(id = h.fields.get(storedIdField) match {
               case Some(List(id: String)) => id
@@ -93,7 +94,7 @@ trait ElastiknnClient[F[_]] extends AutoCloseable {
         }
       }
     }
-    execute(ElastiknnRequests.nearestNeighbors(index, query, k, storedIdField))(handler, implicitly[Manifest[SearchResponse]])
+    execute(ElastiknnRequests.nearestNeighbors(index, query, k, storedIdField))(handler, implicitly[JavaTypeable[SearchResponse]])
   }
 
 }
@@ -118,7 +119,7 @@ object ElastiknnClient {
       implicit val executor: Executor[Future] = Executor.FutureExecutor(ec)
       implicit val functor: Functor[Future] = Functor.FutureFunctor(ec)
       val elasticClient: ElasticClient = ElasticClient(jc)
-      override def execute[T, U](req: T)(implicit handler: Handler[T, U], manifest: Manifest[U]): Future[Response[U]] = {
+      override def execute[T, U](req: T)(implicit handler: Handler[T, U], javaTypeable: JavaTypeable[U]): Future[Response[U]] = {
         val future: Future[Response[U]] = elasticClient.execute(req)
         if (strictFailure) future.flatMap { res =>
           checkResponse(req, res) match {

elastiknn-jmh-benchmarks/src/main/scala/com/klibisz/elastiknn/jmhbenchmarks/FloatVectorOpsBenchmark.scala

@@ -10,8 +10,8 @@ import scala.util.Random
 @State(Scope.Benchmark)
 class FloatVectorOpsBenchmarkState {
   implicit private val rng: Random = new Random(0)
-  val v1 = Vec.DenseFloat.random(999).values
-  val v2 = Vec.DenseFloat.random(999).values
+  val v1: Array[Float] = Vec.DenseFloat.random(999).values
+  val v2: Array[Float] = Vec.DenseFloat.random(999).values
   val panama = new PanamaFloatVectorOps
   val default = new DefaultFloatVectorOps
 }

elastiknn-jmh-benchmarks/src/main/scala/com/klibisz/elastiknn/jmhbenchmarks/HitCounterBenchmarks.scala

@@ -12,7 +12,7 @@ class HitCounterBenchmarksFixtures {
   val numDocs = 60000
   val numHits = 2000
   val initialMapSize = 1000
-  val docs = (1 to numHits).map(_ => rng.nextInt(numDocs)).toArray
+  val docs: Array[Int] = (1 to numHits).map(_ => rng.nextInt(numDocs)).toArray
 }
 
 class HitCounterBenchmarks {

elastiknn-lucene/src/test/scala/com/klibisz/elastiknn/search/ArrayHitCounterSpec.scala

@@ -7,9 +7,9 @@ import scala.util.Random
 
 final class ArrayHitCounterSpec extends AnyFreeSpec with Matchers {
 
-  final class Reference(capacity: Int) extends HitCounter {
+  final class Reference(referenceCapacity: Int) extends HitCounter {
     private val counts = scala.collection.mutable.Map[Int, Short](
-      (0 until capacity).map(_ -> 0.toShort): _*
+      (0 until referenceCapacity).map(_ -> 0.toShort): _*
     )
 
     override def increment(key: Int): Unit = counts.update(key, (counts(key) + 1).toShort)
@@ -22,7 +22,7 @@ final class ArrayHitCounterSpec extends AnyFreeSpec with Matchers {
 
     override def numHits(): Int = counts.values.count(_ > 0)
 
-    override def capacity(): Int = capacity
+    override def capacity(): Int = this.referenceCapacity
 
     override def minKey(): Int = counts.filter(_._2 > 0).keys.min
 

elastiknn-lucene/src/test/scala/org/apache/lucene/search/MatchHashesAndScoreQuerySuite.scala

@@ -14,7 +14,7 @@ import scala.collection.mutable.ArrayBuffer
 class MatchHashesAndScoreQuerySuite extends AnyFunSuite with Matchers with LuceneSupport {
 
   test("empty harness") {
-    indexAndSearch() { (_: IndexWriter) => Assertions.succeed } { (_: IndexReader, _: IndexSearcher) => Assertions.succeed }
+    indexAndSearch() { _ => Assertions.succeed } { (_: IndexReader, _: IndexSearcher) => Assertions.succeed }
   }
 
   test("minimal id example") {

elastiknn-models/src/test/scala/com/klibisz/elastiknn/models/PermutationLshModelSuite.scala

@@ -1,7 +1,7 @@
 package com.klibisz.elastiknn.models
 
 import com.klibisz.elastiknn.api.{Mapping, Vec}
-import com.klibisz.elastiknn.storage.UnsafeSerialization._
+import com.klibisz.elastiknn.storage.ByteBufferSerialization._
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.funsuite.AnyFunSuite
 

elastiknn-plugin-integration-tests/src/test/scala/com/klibisz/elastiknn/DocsWithMissingVectorsSuite.scala

@@ -3,7 +3,6 @@ package com.klibisz.elastiknn
 import com.klibisz.elastiknn.api._
 import com.klibisz.elastiknn.client.ElastiknnRequests
 import com.sksamuel.elastic4s.ElasticDsl._
-import futil.Futil
 import org.scalatest.funsuite.AsyncFunSuite
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.{Inspectors, _}
@@ -38,7 +37,7 @@ class DocsWithMissingVectorsSuite extends AsyncFunSuite with Matchers with Inspe
       _ <- deleteIfExists(index)
       _ <- eknn.createIndex(index)
       _ <- eknn.execute(putMapping(index).rawSource(mappingJsonString))
-      _ <- Futil.traverseSerial((0 until numDocs).grouped(500)) { ids =>
+      _ <- FutureUtils.traverseSerially((0 until numDocs).grouped(500)) { ids =>
         val reqs = ids.map { i =>
           if (i % 2 == 0) ElastiknnRequests.index(index, vecField, if (i == 0) v0 else Vec.DenseFloat.random(dims), idField, s"v$i")
           else indexInto(index).id(s"v$i").fields(Map(idField -> s"v$i"))

elastiknn-plugin-integration-tests/src/test/scala/com/klibisz/elastiknn/DocsWithNestedVectorsSuite.scala

@@ -16,9 +16,9 @@ class DocsWithNestedVectorsSuite extends AsyncFunSuite with Matchers with Inspec
   // https://github.com/alexklibisz/elastiknn/issues/60
   implicit val rng: Random = new Random(0)
   val index = "issue-60"
-  val vec = Vec.DenseFloat.random(10)
-  val mapping = Mapping.DenseFloat(vec.values.length)
-  val nestedFields = Seq(
+  val vec: Vec.DenseFloat = Vec.DenseFloat.random(10)
+  val mapping: Mapping.DenseFloat = Mapping.DenseFloat(vec.values.length)
+  val nestedFields: Seq[String] = Seq(
     "vec",
     "foo.vec",
     "foo.bar.vec",

elastiknn-plugin-integration-tests/src/test/scala/com/klibisz/elastiknn/ExistsQuerySuite.scala

@@ -11,9 +11,9 @@ class ExistsQuerySuite extends AsyncFunSuite with Matchers with ElasticAsyncClie
 
   implicit val rng: Random = new Random(0)
   val (index, field, id) = ("issue-174", "vec", "id")
-  val corpus = Vec.DenseFloat.randoms(128, 99)
-  val ids = corpus.indices.map(i => s"v$i")
-  val mappings = Seq(
+  val corpus: Vector[Vec.DenseFloat] = Vec.DenseFloat.randoms(128, 99)
+  val ids: IndexedSeq[String] = corpus.indices.map(i => s"v$i")
+  val mappings: Seq[Mapping] = Seq(
     Mapping.DenseFloat(corpus.head.dims),
     Mapping.L2Lsh(corpus.head.dims, 50, 1, 2)
   )

elastiknn-plugin-integration-tests/src/test/scala/com/klibisz/elastiknn/FunctionScoreQuerySuite.scala

@@ -4,7 +4,6 @@ import com.klibisz.elastiknn.api._
 import com.klibisz.elastiknn.models.ExactModel
 import com.klibisz.elastiknn.vectors.{FloatVectorOps, PanamaFloatVectorOps}
 import com.sksamuel.elastic4s.ElasticDsl._
-import futil.Futil
 import org.scalatest._
 import org.scalatest.freespec.AsyncFreeSpec
 import org.scalatest.matchers.should.Matchers
@@ -104,7 +103,7 @@ class FunctionScoreQuerySuite extends AsyncFreeSpec with Matchers with Inspector
         _ <- deleteIfExists(index)
         _ <- eknn.createIndex(index)
         _ <- eknn.execute(putMapping(index).rawSource(rawMapping))
-        _ <- Futil.traverseSerial(corpus.grouped(100)) { batch =>
+        _ <- FutureUtils.traverseSerially(corpus.grouped(100)) { batch =>
           val reqs = batch.map { case (id, vec, color) =>
             val docSource = s"""{ "vec": ${XContentCodec.encodeUnsafeToString(vec)}, "color": "$color" }"""
             indexInto(index).id(id).source(docSource)
@@ -211,7 +210,7 @@ class FunctionScoreQuerySuite extends AsyncFreeSpec with Matchers with Inspector
         _ <- deleteIfExists(index)
         _ <- eknn.createIndex(index)
         _ <- eknn.execute(putMapping(index).rawSource(rawMapping))
-        _ <- Futil.traverseSerial(corpus.grouped(100)) { batch =>
+        _ <- FutureUtils.traverseSerially(corpus.grouped(100)) { batch =>
           val reqs = batch.map { case (id, vec, color) =>
             val docSource = s"""{ "id": "$id", "vec": ${XContentCodec.encodeUnsafeToString(vec)}, "color": "$color" }"""
             indexInto(index).id(id).source(docSource)

elastiknn-plugin-integration-tests/src/test/scala/com/klibisz/elastiknn/FutureUtils.scala

@@ -0,0 +1,15 @@
+package com.klibisz.elastiknn
+
+import scala.concurrent.{ExecutionContext, Future}
+
+object FutureUtils {
+
+  def traverseSerially[A, B](in: IterableOnce[A])(f: A => Future[B])(implicit ec: ExecutionContext): Future[Seq[B]] =
+    in.iterator.foldLeft(Future.successful(Vector.empty[B])) { case (accFuture, next) =>
+      for {
+        acc <- accFuture
+        result <- f(next)
+      } yield acc :+ result
+    }
+
+}

elastiknn-plugin-integration-tests/src/test/scala/com/klibisz/elastiknn/MixedIndexSearchDeleteSuite.scala

@@ -1,9 +1,7 @@
 package com.klibisz.elastiknn
 
 import com.klibisz.elastiknn.api._
-import com.klibisz.elastiknn._
 import com.sksamuel.elastic4s.ElasticDsl._
-import futil.Futil
 import org.scalatest._
 import org.scalatest.funsuite.AsyncFunSuite
 import org.scalatest.matchers.should.Matchers
@@ -37,7 +35,7 @@ class MixedIndexSearchDeleteSuite extends AsyncFunSuite with Matchers with Inspe
 
         // Delete the top five vectors.
         deletedIdxs = s1.result.hits.hits.take(5).map(_.id.drop(1).toInt).toSeq
-        _ <- Futil.traverseParN(8)(deletedIdxs.map(ids.apply).map(deleteById(index, _)))(eknn.execute(_))
+        _ <- FutureUtils.traverseSerially(deletedIdxs.map(ids.apply).map(deleteById(index, _)))(eknn.execute(_))
         _ <- eknn.execute(refreshIndex(index))
         c2 <- eknn.execute(count(index))
         _ = c2.result.count shouldBe (corpus.length - deletedIdxs.length)