[JSON-API] Migrating tests to use sandbox next (#11034)

* Changes to use sandbox next for our integration tests

CHANGELOG_BEGIN
CHANGELOG_END

* remove sandbox classic dependency for HttpServiceTestFixture and perf tests

* rely on sandbox next fixture test class

* add missing dependencies for http-json-oracle

* changes based on code review comments

* Add tag to skip test case for scala_2_12 and also add a jdbc backend for sandbox spun up for perf tests

* Reduce size of contracts for archiving test

ledger-service/http-json-oracle/BUILD.bazel

@@ -51,13 +51,17 @@ da_scala_test(
         "//daml-lf/transaction",
         "//language-support/scala/bindings-akka",
         "//ledger-api/rs-grpc-bridge",
+        "//ledger-api/testing-utils",
         "//ledger-service/db-backend",
         "//ledger-service/http-json:http-json-ee",
         "//ledger-service/http-json:integration-tests-lib-ee",
         "//ledger-service/http-json-cli:ee",
         "//ledger-service/http-json-testing:ee",
         "//ledger-service/jwt",
         "//ledger-service/utils",
+        "//ledger/sandbox:sandbox-scala-tests-lib",
+        "//ledger/sandbox-common",
+        "//ledger/sandbox-common:sandbox-common-scala-tests-lib",
         "//libs-scala/db-utils",
         "//libs-scala/oracle-testing",
         "//libs-scala/ports",

ledger-service/http-json-testing/BUILD.bazel

@@ -43,6 +43,7 @@ hj_scalacopts = lf_scalacopts + [
             "//bazel_tools/runfiles:scala_runfiles",
             "//language-support/scala/bindings-akka",
             "//ledger-api/rs-grpc-bridge",
+            "//ledger-api/testing-utils",
             "@maven//:org_scalatest_scalatest_compatible",
             "//ledger-service/http-json:http-json-{}".format(edition),
             "//ledger-service/http-json-cli:{}".format(edition),
@@ -53,13 +54,17 @@ hj_scalacopts = lf_scalacopts + [
             "//ledger/ledger-api-auth",
             "//ledger/ledger-api-common",
             "//ledger/ledger-configuration",
+            "//ledger/ledger-resources",
             "//ledger/metrics",
             "//ledger/participant-integration-api",
-            "//ledger/sandbox-classic",
+            "//ledger/sandbox",
+            "//ledger/sandbox:sandbox-scala-tests-lib",
             "//ledger/sandbox-common",
+            "//ledger/sandbox-common:sandbox-common-scala-tests-lib",
             "//libs-scala/contextualized-logging",
             "//libs-scala/db-utils",
             "//libs-scala/ports",
+            "//libs-scala/resources",
             "@maven//:io_dropwizard_metrics_metrics_core",
         ],
     )

ledger-service/http-json-testing/src/main/scala/com/daml/http/HttpServiceTestFixture.scala

@@ -37,13 +37,14 @@ import com.daml.ledger.client.configuration.{
   LedgerIdRequirement,
 }
 import com.daml.ledger.client.withoutledgerid.{LedgerClient => DamlLedgerClient}
+import com.daml.ledger.resources.ResourceContext
 import com.daml.logging.LoggingContextOf
 import com.daml.metrics.Metrics
 import com.daml.platform.apiserver.SeedService.Seeding
 import com.daml.platform.common.LedgerIdMode
-import com.daml.platform.sandbox
-import com.daml.platform.sandbox.SandboxServer
+import com.daml.platform.sandbox.SandboxBackend
 import com.daml.platform.sandbox.config.SandboxConfig
+import com.daml.platform.sandboxnext.Runner
 import com.daml.platform.services.time.TimeProviderType
 import com.daml.ports.Port
 import com.typesafe.scalalogging.LazyLogging
@@ -150,19 +151,35 @@ object HttpServiceTestFixture extends LazyLogging with Assertions with Inside {
       useTls: UseTls = UseTls.NoTls,
       authService: Option[AuthService] = None,
   )(testFn: (Port, DamlLedgerClient, LedgerId) => Future[A])(implicit
-      mat: Materializer,
       aesf: ExecutionSequencerFactory,
       ec: ExecutionContext,
   ): Future[A] = {
 
     val ledgerId = LedgerId(testName)
     val applicationId = ApplicationId(testName)
+    implicit val resourceContext: ResourceContext = ResourceContext(ec)
 
     val ledgerF = for {
+      urlResource <- Future(
+        SandboxBackend.H2Database.owner
+          .map(info => Some(info.jdbcUrl))
+          .acquire()
+      )
+      jdbcUrl <- urlResource.asFuture
       ledger <- Future(
-        new SandboxServer(ledgerConfig(Port.Dynamic, dars, ledgerId, authService, useTls), mat)
+        new Runner(
+          ledgerConfig(
+            Port.Dynamic,
+            dars,
+            ledgerId,
+            useTls = useTls,
+            authService = authService,
+            jdbcUrl = jdbcUrl,
+          )
+        )
+          .acquire()
       )
-      port <- ledger.portF
+      port <- ledger.asFuture
     } yield (ledger, port)
 
     val clientF: Future[DamlLedgerClient] = for {
@@ -179,11 +196,12 @@ object HttpServiceTestFixture extends LazyLogging with Assertions with Inside {
       a <- testFn(ledgerPort, client, ledgerId)
     } yield a
 
-    fa.onComplete { _ =>
-      ledgerF.foreach(_._1.close())
+    fa.transformWith { ta =>
+      ledgerF
+        .flatMap(_._1.release())
+        .fallbackTo(Future.unit)
+        .transform(_ => ta)
     }
-
-    fa
   }
 
   private def ledgerConfig(
@@ -192,12 +210,15 @@ object HttpServiceTestFixture extends LazyLogging with Assertions with Inside {
       ledgerId: LedgerId,
       authService: Option[AuthService],
       useTls: UseTls,
+      jdbcUrl: Option[String],
   ): SandboxConfig =
-    sandbox.DefaultConfig.copy(
+    SandboxConfig.defaultConfig.copy(
       port = ledgerPort,
       damlPackages = dars,
+      jdbcUrl = jdbcUrl,
       timeProviderType = Some(TimeProviderType.WallClock),
       tlsConfig = if (useTls) Some(serverTlsConfig) else None,
+      engineMode = SandboxConfig.EngineMode.Dev,
       ledgerIdMode = LedgerIdMode.Static(ledgerId),
       authService = authService,
       seeding = Some(Seeding.Weak),
@@ -271,8 +292,8 @@ object HttpServiceTestFixture extends LazyLogging with Assertions with Inside {
     }
   }
 
-  private val serverTlsConfig = TlsConfiguration(enabled = true, serverCrt, serverPem, caCrt)
-  private val clientTlsConfig = TlsConfiguration(enabled = true, clientCrt, clientPem, caCrt)
+  final val serverTlsConfig = TlsConfiguration(enabled = true, serverCrt, serverPem, caCrt)
+  final val clientTlsConfig = TlsConfiguration(enabled = true, clientCrt, clientPem, caCrt)
   private val noTlsConfig = TlsConfiguration(enabled = false, None, None, None)
 
   def jwtForParties(actAs: List[String], readAs: List[String], ledgerId: String) = {

ledger-service/http-json-testing/src/main/scala/com/daml/http/util/SandboxTestLedger.scala

@@ -0,0 +1,78 @@
+// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.daml.http.util
+
+import com.daml.grpc.adapter.ExecutionSequencerFactory
+import com.daml.http.HttpServiceTestFixture.{UseTls, clientTlsConfig, serverTlsConfig}
+import com.daml.ledger.api.domain.LedgerId
+import com.daml.ledger.client.configuration.{
+  CommandClientConfiguration,
+  LedgerClientConfiguration,
+  LedgerIdRequirement,
+}
+import com.daml.ports.Port
+import com.daml.ledger.client.withoutledgerid.{LedgerClient => DamlLedgerClient}
+import com.daml.platform.apiserver.SeedService.Seeding
+import com.daml.platform.common.LedgerIdMode
+import com.daml.platform.sandbox.config.SandboxConfig
+import com.daml.platform.sandboxnext.SandboxNextFixture
+import com.daml.platform.services.time.TimeProviderType
+import org.scalatest.Suite
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait SandboxTestLedger extends SandboxNextFixture {
+  self: Suite =>
+
+  protected def testId: String
+
+  def useTls: UseTls
+
+  def ledgerId = LedgerId(testId)
+
+  override protected def config: SandboxConfig = SandboxConfig.defaultConfig.copy(
+    port = Port.Dynamic,
+    damlPackages = packageFiles,
+    timeProviderType = Some(TimeProviderType.WallClock),
+    tlsConfig = if (useTls) Some(serverTlsConfig) else None,
+    ledgerIdMode = LedgerIdMode.Static(ledgerId),
+    authService = authService,
+    scenario = scenario,
+    engineMode = SandboxConfig.EngineMode.Dev,
+    seeding = Some(Seeding.Weak),
+  )
+
+  def clientCfg(token: Option[String], testName: String): LedgerClientConfiguration =
+    LedgerClientConfiguration(
+      applicationId = testName,
+      ledgerIdRequirement = LedgerIdRequirement.none,
+      commandClient = CommandClientConfiguration.default,
+      sslContext = if (useTls) clientTlsConfig.client() else None,
+      token = token,
+    )
+
+  def usingLedger[A](testName: String, token: Option[String] = None)(
+      testFn: (Port, DamlLedgerClient, LedgerId) => Future[A]
+  )(implicit
+      esf: ExecutionSequencerFactory,
+      ec: ExecutionContext,
+  ): Future[A] = {
+
+    val clientF: Future[DamlLedgerClient] = for {
+      ledgerPort <- Future(serverPort)
+    } yield DamlLedgerClient.singleHost(
+      "localhost",
+      ledgerPort.value,
+      clientCfg(token, testName),
+    )(ec, esf)
+
+    val fa: Future[A] = for {
+      ledgerPort <- Future(serverPort)
+      client <- clientF
+      a <- testFn(ledgerPort, client, ledgerId)
+    } yield a
+
+    fa
+  }
+}

ledger-service/http-json/BUILD.bazel

@@ -12,7 +12,7 @@ load(
 )
 load("//rules_daml:daml.bzl", "daml_compile")
 load("@os_info//:os_info.bzl", "is_windows")
-load("@scala_version//:index.bzl", "scala_version_suffix")
+load("@scala_version//:index.bzl", "scala_major_version", "scala_version_suffix")
 
 hj_scalacopts = lf_scalacopts + [
     "-P:wartremover:traverser:org.wartremover.warts.NonUnitStatements",
@@ -283,7 +283,12 @@ alias(
             "//daml-lf/transaction-test-lib",
             "//language-support/scala/bindings-akka",
             "//ledger-api/rs-grpc-bridge",
+            "//ledger-api/testing-utils",
+            "//ledger/ledger-resources",
             "//ledger/metrics",
+            "//ledger/sandbox:sandbox-scala-tests-lib",
+            "//ledger/sandbox-common",
+            "//ledger/sandbox-common:sandbox-common-scala-tests-lib",
             "//ledger-service/http-json-cli:{}".format(edition),
             "//ledger-service/http-json-testing:{}".format(edition),
             "//ledger-service/db-backend",
@@ -326,6 +331,10 @@ alias(
             "src/it/scala/**/*.scala",
             "src/it/edition/{}/**/*.scala".format(edition),
         ]),
+        args = [
+            "-l",
+            "skip_scala_2_12",
+        ] if scala_major_version == "2.12" else [],
         data = [
             ":Account.dar",
             "//docs:quickstart-model.dar",
@@ -369,17 +378,23 @@ alias(
             "//daml-lf/transaction-test-lib",
             "//language-support/scala/bindings-akka",
             "//ledger-api/rs-grpc-bridge",
+            "//ledger-api/testing-utils",
             "//ledger-service/db-backend",
+            "//ledger/ledger-api-auth",
+            "//ledger/ledger-resources",
             "//ledger/metrics",
+            "//ledger/sandbox:sandbox-scala-tests-lib",
+            "//ledger/sandbox-common",
+            "//ledger/sandbox-common:sandbox-common-scala-tests-lib",
             "//ledger-service/http-json-cli:{}".format(edition),
             "//ledger-service/http-json-testing:{}".format(edition),
             "//ledger-service/jwt",
             "//ledger-service/utils",
-            "//ledger/ledger-api-auth",
             "//libs-scala/contextualized-logging",
             "//libs-scala/db-utils",
             "//libs-scala/ports",
             "//libs-scala/postgresql-testing",
+            "//libs-scala/resources",
             "//libs-scala/scala-utils",
             "//runtime-components/non-repudiation",
             "//runtime-components/non-repudiation-postgresql",

ledger-service/http-json/src/it/scala/http/AuthorizationTest.scala

@@ -8,11 +8,14 @@ import akka.actor.ActorSystem
 import akka.stream.Materializer
 import com.daml.bazeltools.BazelRunfiles.rlocation
 import com.daml.grpc.adapter.{AkkaExecutionSequencerPool, ExecutionSequencerFactory}
+import com.daml.http.HttpServiceTestFixture.UseTls
 import com.daml.http.util.TestUtil.requiredFile
 import com.daml.http.util.Logging.instanceUUIDLogCtx
+import com.daml.http.util.SandboxTestLedger
 import com.daml.jwt.domain.Jwt
 import com.daml.ledger.api.auth.{AuthServiceStatic, Claim, ClaimPublic, ClaimSet}
 import com.daml.ledger.api.domain.LedgerId
+import com.daml.ledger.api.testing.utils.SuiteResourceManagementAroundAll
 import com.daml.ledger.client.withoutledgerid.{LedgerClient => DamlLedgerClient}
 import org.scalatest.BeforeAndAfterAll
 import org.scalatest.flatspec.AsyncFlatSpec
@@ -22,12 +25,18 @@ import org.slf4j.LoggerFactory
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.control.NonFatal
 
-final class AuthorizationTest extends AsyncFlatSpec with BeforeAndAfterAll with Matchers {
+final class AuthorizationTest
+    extends AsyncFlatSpec
+    with BeforeAndAfterAll
+    with Matchers
+    with SandboxTestLedger
+    with SuiteResourceManagementAroundAll {
 
   private val dar = requiredFile(rlocation("docs/quickstart-model.dar"))
     .fold(e => throw new IllegalStateException(e), identity)
 
-  private val testId: String = this.getClass.getSimpleName
+  protected val testId: String = this.getClass.getSimpleName
+  override def useTls = UseTls.NoTls
 
   implicit val asys: ActorSystem = ActorSystem(testId)
   implicit val mat: Materializer = Materializer(asys)
@@ -43,6 +52,9 @@ final class AuthorizationTest extends AsyncFlatSpec with BeforeAndAfterAll with
 
   private val accessTokenFile = Files.createTempFile("Extractor", "AuthSpec")
 
+  override def authService = mockedAuthService
+  override def packageFiles = List(dar)
+
   override protected def afterAll(): Unit = {
     super.afterAll()
     try {
@@ -55,11 +67,11 @@ final class AuthorizationTest extends AsyncFlatSpec with BeforeAndAfterAll with
     }
   }
 
-  protected def withLedger[A](testFn: DamlLedgerClient => LedgerId => Future[A]): Future[A] =
-    HttpServiceTestFixture
-      .withLedger[A](List(dar), testId, Option(publicToken), authService = mockedAuthService) {
-        case (_, client, ledgerId) => testFn(client)(ledgerId)
-      }
+  protected def withLedger[A](testFn: DamlLedgerClient => LedgerId => Future[A]): Future[A] = {
+    usingLedger[A](testId, Some(publicToken)) { case (_, client, ledgerId) =>
+      testFn(client)(ledgerId)
+    }
+  }
 
   private def packageService(client: DamlLedgerClient): PackageService =
     new PackageService(HttpService.doLoad(client.packageClient))

ledger-service/http-json/src/it/scala/http/HttpServiceWithPostgresIntTest.scala

@@ -20,11 +20,14 @@ class HttpServiceWithPostgresIntTest
   override def wsConfig: Option[WebsocketConfig] = None
 
   "query persists all active contracts" in withHttpService { (uri, encoder, _, _) =>
+    val (party, headers) = getUniquePartyAndAuthHeaders("Alice")
+    val searchDataSet = genSearchDataSet(party)
     searchExpectOk(
       searchDataSet,
       jsObject("""{"templateIds": ["Iou:Iou"], "query": {"currency": "EUR"}}"""),
       uri,
       encoder,
+      headers,
     ).flatMap { searchResult: List[domain.ActiveContract[JsValue]] =>
       discard { searchResult should have size 2 }
       discard { searchResult.map(getField("currency")) shouldBe List.fill(2)(JsString("EUR")) }

ledger-service/http-json/src/it/scala/http/WebsocketServiceOffsetTickIntTest.scala

@@ -3,13 +3,14 @@
 
 package com.daml.http
 
-import com.daml.http.HttpServiceTestFixture.UseTls
+import com.daml.http.HttpServiceTestFixture.{UseTls, jwtForParties}
 import com.daml.http.dbbackend.JdbcConfig
 import com.typesafe.scalalogging.StrictLogging
 import org.scalatest._
 import org.scalatest.freespec.AsyncFreeSpec
 import org.scalatest.matchers.should.Matchers
 import scalaz.\/-
+import scalaz.syntax.tag._
 
 import scala.concurrent.duration._
 
@@ -49,9 +50,10 @@ class WebsocketServiceOffsetTickIntTest
 
   "Given non-empty ACS, JSON API should emit ACS block and after it only absolute offset ticks" in withHttpService {
     (uri, _, _, _) =>
+      val (party, headers) = getUniquePartyAndAuthHeaders("Alice")
       for {
-        _ <- initialIouCreate(uri)
-
+        _ <- initialIouCreate(uri, party, headers)
+        jwt = jwtForParties(List(party.unwrap), List(), testId)
         msgs <- singleClientQueryStream(jwt, uri, """{"templateIds": ["Iou:Iou"]}""")
           .take(10)
           .runWith(collectResultsAsTextMessage)