From df59f3fe8e1448435ee33303888c0b516dbba7e0 Mon Sep 17 00:00:00 2001 From: Moritz Kiefer Date: Tue, 28 Sep 2021 10:07:56 +0200 Subject: [PATCH 1/9] Fix Navigator dependabot alerts (#11044) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I used a selective override for strip-ansi because html-webpack-plugin cannot handle 7.0.0 and I don’t want to downgrade it in other places where we are already using 7.0.0. changelog_begin changelog_end --- navigator/frontend/package.json | 3 ++- navigator/frontend/yarn.lock | 36 +++++++++++---------------------- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/navigator/frontend/package.json b/navigator/frontend/package.json index 11c9e9431611..314908c3c0c3 100644 --- a/navigator/frontend/package.json +++ b/navigator/frontend/package.json @@ -98,6 +98,7 @@ "**/ws": "^7.4.6", "**/css-what": "^5.0.1", "**/glob-parent": "^6.0.0", - "**/set-value": "^4.0.1" + "**/set-value": "^4.0.1", + "html-webpack-plugin/**/strip-ansi": "^6.0.0" } } diff --git a/navigator/frontend/yarn.lock b/navigator/frontend/yarn.lock index 6dc33181c416..02024980860d 100644 --- a/navigator/frontend/yarn.lock +++ b/navigator/frontend/yarn.lock @@ -803,15 +803,10 @@ ansi-html-community@^0.0.8: resolved "https://registry.yarnpkg.com/ansi-html-community/-/ansi-html-community-0.0.8.tgz#69fbc4d6ccbe383f9736934ae34c3f8290f1bf41" integrity sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw== -ansi-regex@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" - integrity sha1-w7M6te42DYbg5ijwRorn7yfWVN8= - -ansi-regex@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" - integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== +ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== ansi-regex@^6.0.1: version "6.0.1" @@ -3511,9 +3506,9 @@ npm-run-path@^4.0.1: path-key "^3.0.0" nth-check@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.0.0.tgz#1bb4f6dac70072fc313e8c9cd1417b5074c0a125" - integrity sha512-i4sc/Kj8htBrAiH1viZ0TgU8Y5XqCaV/FziYK6TBczxmeKm3AEFWqqF3195yKudrarqy7Zu80Ra5dobFjn9X/Q== + version "2.0.1" + resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.0.1.tgz#2efe162f5c3da06a28959fbd3db75dbeea9f0fc2" + integrity sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w== dependencies: boolbase "^1.0.0" @@ -4815,19 +4810,12 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -strip-ansi@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" - integrity sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8= - dependencies: - ansi-regex "^2.0.0" - -strip-ansi@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" - integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== +strip-ansi@^3.0.1, strip-ansi@^6.0.0: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== dependencies: - ansi-regex "^5.0.0" + ansi-regex "^5.0.1" strip-ansi@^7.0.0: version "7.0.1" From 57a15972c75bbd92d46982f97aa35b63297c0714 Mon Sep 17 00:00:00 2001 From: Marton Nagy Date: Tue, 28 Sep 2021 10:25:42 +0200 Subject: [PATCH 2/9] Setting timeoutToleranceMillis to 10 minutes to prevent flakiness (#11043) CHANGELOG_BEGIN CHANGELOG_END --- .../suite/scala/platform/indexer/ha/HaCoordinatorSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/participant-integration-api/src/test/suite/scala/platform/indexer/ha/HaCoordinatorSpec.scala b/ledger/participant-integration-api/src/test/suite/scala/platform/indexer/ha/HaCoordinatorSpec.scala index f51f50beb4bd..8217a57d77a1 100644 --- a/ledger/participant-integration-api/src/test/suite/scala/platform/indexer/ha/HaCoordinatorSpec.scala +++ b/ledger/participant-integration-api/src/test/suite/scala/platform/indexer/ha/HaCoordinatorSpec.scala @@ -35,7 +35,7 @@ class HaCoordinatorSpec private val workerLockAcquireRetryMillis = 20L private val mainLockCheckerPeriodMillis = 20L private val timeoutToleranceMillis = - 1000L // unfortunately this needs to be a insanely big tolerance, not to render the test flaky. under normal circumstances this should pass with +5 millis + 600000L // unfortunately this needs to be a insanely big tolerance, not to render the test flaky. under normal circumstances this should pass with +5 millis private val mainLockId = 10 private val main = TestLockId(mainLockId) From 03203b736030e832c5df92f786234e4965d3cc08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Ackerman?= <6054733+akrmn@users.noreply.github.com> Date: Tue, 28 Sep 2021 10:48:48 +0200 Subject: [PATCH 3/9] Define encoding/decoding for module imports (#11036) * Define encoding/decoding for module imports First step towards closing #10773 changelog_begin changelog_end * Update compiler/damlc/daml-lf-conversion/src/DA/Daml/LFConversion/MetadataEncoding.hs Co-authored-by: Sofia Faro --- compiler/damlc/daml-lf-conversion/BUILD.bazel | 1 + .../DA/Daml/LFConversion/MetadataEncoding.hs | 52 +++++++++++++++++++ .../test/DA/Daml/LFConversion/Tests.hs | 26 ++++++++++ 3 files changed, 79 insertions(+) diff --git a/compiler/damlc/daml-lf-conversion/BUILD.bazel b/compiler/damlc/daml-lf-conversion/BUILD.bazel index a7f3ba1d2141..de4ad331edbd 100644 --- a/compiler/damlc/daml-lf-conversion/BUILD.bazel +++ b/compiler/damlc/daml-lf-conversion/BUILD.bazel @@ -47,6 +47,7 @@ da_haskell_test( srcs = glob(["test/**/*.hs"]), hackage_deps = [ "base", + "containers", "either", "ghc-lib-parser", "ghc-lib", diff --git a/compiler/damlc/daml-lf-conversion/src/DA/Daml/LFConversion/MetadataEncoding.hs b/compiler/damlc/daml-lf-conversion/src/DA/Daml/LFConversion/MetadataEncoding.hs index 41879e7045af..48dbfd21ad32 100644 --- a/compiler/damlc/daml-lf-conversion/src/DA/Daml/LFConversion/MetadataEncoding.hs +++ b/compiler/damlc/daml-lf-conversion/src/DA/Daml/LFConversion/MetadataEncoding.hs @@ -1,6 +1,7 @@ -- Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -- SPDX-License-Identifier: Apache-2.0 +{-# LANGUAGE BlockArguments #-} {-# LANGUAGE PatternSynonyms #-} -- | Encoding/decoding of metadata (i.e. non-semantically-relevant bindings) in LF, @@ -22,11 +23,15 @@ module DA.Daml.LFConversion.MetadataEncoding , encodeOverlapMode , decodeOverlapMode , mkMetadataStub + , moduleImportsName + , encodeModuleImports + , decodeModuleImports ) where import Safe (readMay) import Control.Monad (guard, liftM2) import Data.List (sortOn) +import qualified Data.Set as S import qualified Data.Text as T import qualified "ghc-lib-parser" BasicTypes as GHC @@ -104,6 +109,10 @@ minimalName (LF.TypeSynName xs) = LF.ExprValName ("$$minimal" <> T.concat xs) pattern TEncodedStr :: T.Text -> LF.Type pattern TEncodedStr x = LF.TStruct [(LF.FieldName x, LF.TUnit)] +decodeText :: LF.Type -> Maybe T.Text +decodeText (TEncodedStr x) = Just x +decodeText _ = Nothing + pattern TEncodedCon :: T.Text -> LF.Type -> LF.Type pattern TEncodedCon a b = LF.TStruct [(LF.FieldName a, b)] @@ -153,6 +162,49 @@ decodeOverlapMode = \case ] _ -> Nothing +-------------------------- +-- INSTANCE PROPAGATION -- +-------------------------- +moduleImportsName :: LF.ExprValName +moduleImportsName = LF.ExprValName "$$imports" + +encodeModuleImports :: S.Set (LF.Qualified ()) -> LF.Type +encodeModuleImports = encodeTypeList encodeModuleImport . S.toList + +encodeModuleImport :: LF.Qualified () -> LF.Type +encodeModuleImport q = + encodeTypeList id + [ encodePackageRef (LF.qualPackage q) + , encodeModuleName (LF.qualModule q) + ] + +encodePackageRef :: LF.PackageRef -> LF.Type +encodePackageRef = \case + LF.PRSelf -> LF.TUnit + LF.PRImport (LF.PackageId packageId) -> TEncodedStr packageId + +encodeModuleName :: LF.ModuleName -> LF.Type +encodeModuleName (LF.ModuleName components) = + encodeTypeList TEncodedStr components + +decodeModuleImports :: LF.Type -> Maybe (S.Set (LF.Qualified ())) +decodeModuleImports = fmap S.fromList . decodeTypeList decodeModuleImport + +decodeModuleImport :: LF.Type -> Maybe (LF.Qualified ()) +decodeModuleImport x = do + [p, m] <- decodeTypeList Just x + packageRef <- decodePackageRef p + moduleName <- decodeModuleName m + pure (LF.Qualified packageRef moduleName ()) + +decodePackageRef :: LF.Type -> Maybe LF.PackageRef +decodePackageRef = \case + LF.TUnit -> pure LF.PRSelf + TEncodedStr packageId -> pure (LF.PRImport (LF.PackageId packageId)) + _ -> Nothing + +decodeModuleName :: LF.Type -> Maybe LF.ModuleName +decodeModuleName = fmap LF.ModuleName . decodeTypeList decodeText --------------------- -- STUB GENERATION -- diff --git a/compiler/damlc/daml-lf-conversion/test/DA/Daml/LFConversion/Tests.hs b/compiler/damlc/daml-lf-conversion/test/DA/Daml/LFConversion/Tests.hs index 4776406a89ee..673eb0066c8f 100644 --- a/compiler/damlc/daml-lf-conversion/test/DA/Daml/LFConversion/Tests.hs +++ b/compiler/damlc/daml-lf-conversion/test/DA/Daml/LFConversion/Tests.hs @@ -10,6 +10,7 @@ import Test.Tasty.HUnit import Data.Either.Combinators (whenLeft, whenRight) import Data.Maybe (isNothing) import Data.Ratio +import qualified Data.Set as S import qualified Data.Text as T import DA.Daml.LFConversion @@ -58,8 +59,33 @@ metadataEncodingTests = testGroup "MetadataEncoding" , ("overlaps", GHC.Overlaps GHC.NoSourceText) , ("incoherent", GHC.Incoherent GHC.NoSourceText) ] + , roundtripTests "module imports" encodeModuleImports decodeModuleImports + [ ("()", S.empty) + , ("(Foo.Bar)" + , S.fromList + [ mkImport Nothing ["Foo", "Bar"]]) + , ("(\"foo\" Foo.Bar)" + , S.fromList + [ mkImport (Just "foo") ["Foo", "Bar"]]) + , ("(Foo.Bar, Baz.Qux.Florp)" + , S.fromList + [ mkImport Nothing ["Foo", "Bar"] + , mkImport Nothing ["Baz", "Qux", "Florp"]]) + , ("(\"foo\" Foo.Bar, \"baz\" Baz.Qux.Florp)" + , S.fromList + [ mkImport (Just "foo") ["Foo", "Bar"] + , mkImport (Just "baz") ["Baz", "Qux", "Florp"]]) + ] ] +mkImport :: Maybe T.Text -> [T.Text] -> LF.Qualified () +mkImport mPackage moduleComponents = + LF.Qualified + { qualPackage = maybe LF.PRSelf (LF.PRImport . LF.PackageId) mPackage + , qualModule = LF.ModuleName moduleComponents + , qualObject = () + } + roundtripTests :: (Eq a) => String -> (a -> b) -> (b -> Maybe a) -> [(String, a)] -> TestTree roundtripTests groupName encode decode examples = roundtripTestsPartial groupName (Just . encode) decode [] examples From 5e43f8c703e815fac702adcfec759c5e1dfb0ac9 Mon Sep 17 00:00:00 2001 From: Gary Verhaegen Date: Tue, 28 Sep 2021 11:06:52 +0200 Subject: [PATCH 4/9] es: drop jobs-* indices (#10857) We are currently ingesting Bazel events in two forms: In the `events-*` indices, each Bazel event is recorded as a separate ES object, with the corresponding job name as a field that can serve to aggregate all of the events for a given job. In the `jobs-*` indices, each job is ingested as a single (composite) ES object, with the individual events as elements in a list-type field. When I set up the cluster, I wasn't sure which one would be more useful, so I included both. We now have a bit more usage experience and it turns out the `events-*` form is the only one we use, so I think we should stop ingesting evrything twice and from now on create only the `events-*` ones. CHANGELOG_BEGIN CHANGELOG_END --- infra/es_cluster.tf | 37 ++++--------------------------------- 1 file changed, 4 insertions(+), 33 deletions(-) diff --git a/infra/es_cluster.tf b/infra/es_cluster.tf index d99300899a64..c9625557f181 100644 --- a/infra/es_cluster.tf +++ b/infra/es_cluster.tf @@ -738,7 +738,7 @@ emit_build_events() { jq -c \ --slurpfile job_md "$job/job-md.json" \ --arg cmd "$cmd" \ - --arg index "$(index "$job" events)" \ + --arg index "$(index "$job")" \ --arg job "$job" \ < "$file" \ ' @@ -748,20 +748,6 @@ emit_build_events() { buildEvent: . } ' - jq -c \ - --slurpfile job_md "$job/job-md.json" \ - --arg cmd "$cmd" \ - --arg index "$(index "$job" jobs)" \ - --arg job "$job" \ - < "$file" \ - --slurp \ - ' - { index: { _index: $index, _id: ($job + "-" + $cmd + "-events") } }, - { job: $job_md[0], - command: { name: $cmd }, - buildEvent: . - } - ' } emit_trace_events() { @@ -772,7 +758,7 @@ emit_trace_events() { jq -c \ --slurpfile job_md "$job/job-md.json" \ --arg cmd "$cmd" \ - --arg index "$(index "$job" events)" \ + --arg index "$(index "$job")" \ --arg job "$job" \ < "$file" \ ' @@ -784,19 +770,6 @@ emit_trace_events() { traceEvent: .value } ' - jq -c \ - --slurpfile job_md "$job/job-md.json" \ - --arg cmd "$cmd" \ - --arg index "$(index "$job" jobs)" \ - --arg job "$job" \ - < "$file" \ - ' - { index: { _index: $index, _id: ($job + "-" + $cmd + "-profile") } }, - { job: $job_md[0], - command: { name: $cmd }, - traceEvent: .traceEvents - } - ' } bulk_upload() { @@ -863,8 +836,7 @@ push() { index() { local job prefix job="$1" - prefix="$2" - echo "$prefix-$(echo $job | cut -c1-10)" + echo "events-$(echo $job | cut -c1-10)" } pid=$$ @@ -895,8 +867,7 @@ for tar in $todo; do job=$(basename $${tar%.tar.gz}) cd $(dirname $tar) if ! [ -f $DONE/$job ]; then - ensure_index "$job" "$(index "$job" jobs)" - ensure_index "$job" "$(index "$job" events)" + ensure_index "$job" "$(index "$job")" tar --force-local -x -z -f "$(basename "$tar")" patch "$job" push "$job" From b7daa5f7d8187c4d918b6c060dc6c541160820fd Mon Sep 17 00:00:00 2001 From: Moritz Kiefer Date: Tue, 28 Sep 2021 11:12:14 +0200 Subject: [PATCH 5/9] Address remaining dependabot alerts (#11045) * Address remaining dependabot alerts changelog_begin changelog_end * downgrade string-width 5.0.0 is too new for our setup changelog_begin changelog_end * . changelog_begin changelog_end --- compiler/daml-extension/yarn.lock | 6 ++-- language-support/ts/packages/package.json | 3 +- language-support/ts/packages/yarn.lock | 44 +++++++++++------------ 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/compiler/daml-extension/yarn.lock b/compiler/daml-extension/yarn.lock index fd569eabfa1d..982699dc30ed 100644 --- a/compiler/daml-extension/yarn.lock +++ b/compiler/daml-extension/yarn.lock @@ -403,9 +403,9 @@ node-fetch@^2.6.1: integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== nth-check@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.0.0.tgz#1bb4f6dac70072fc313e8c9cd1417b5074c0a125" - integrity sha512-i4sc/Kj8htBrAiH1viZ0TgU8Y5XqCaV/FziYK6TBczxmeKm3AEFWqqF3195yKudrarqy7Zu80Ra5dobFjn9X/Q== + version "2.0.1" + resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.0.1.tgz#2efe162f5c3da06a28959fbd3db75dbeea9f0fc2" + integrity sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w== dependencies: boolbase "^1.0.0" diff --git a/language-support/ts/packages/package.json b/language-support/ts/packages/package.json index 84974051c2b3..e55334e83868 100644 --- a/language-support/ts/packages/package.json +++ b/language-support/ts/packages/package.json @@ -31,6 +31,7 @@ "marked": "^2.0.0", "**/y18n": "^4.0.1", "**/@types/react-test-renderer": "^16.9.3", - "**/hosted-git-info": "^4.0.2" + "**/hosted-git-info": "^4.0.2", + "**/string-width": "^4.2.3" } } diff --git a/language-support/ts/packages/yarn.lock b/language-support/ts/packages/yarn.lock index 08a354d425cd..e38c7d245795 100644 --- a/language-support/ts/packages/yarn.lock +++ b/language-support/ts/packages/yarn.lock @@ -1079,6 +1079,11 @@ ansi-regex@^5.0.0: resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== +ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== + ansi-styles@^3.2.0, ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" @@ -1509,11 +1514,6 @@ emittery@^0.8.1: resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.8.1.tgz#bb23cc86d03b30aa75a7f734819dee2e1ba70860" integrity sha512-uDfvUjVrfGJJhymx/kz6prltenw1u7WrCg1oa94zYY8xxVpLLUu045LAT0dhDZdXG58/EpPL/5kA180fQ/qudg== -emoji-regex@^7.0.1: - version "7.0.3" - resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156" - integrity sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA== - emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" @@ -3294,25 +3294,16 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -string-width@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-3.1.0.tgz#22767be21b62af1081574306f69ac51b62203961" - integrity sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w== - dependencies: - emoji-regex "^7.0.1" - is-fullwidth-code-point "^2.0.0" - strip-ansi "^5.1.0" - -string-width@^4.1.0, string-width@^4.2.0: - version "4.2.2" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5" - integrity sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA== +string-width@^3.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== dependencies: emoji-regex "^8.0.0" is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.0" + strip-ansi "^6.0.1" -strip-ansi@^5.1.0, strip-ansi@^5.2.0: +strip-ansi@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae" integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA== @@ -3326,6 +3317,13 @@ strip-ansi@^6.0.0: dependencies: ansi-regex "^5.0.0" +strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-bom@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878" @@ -3425,9 +3423,9 @@ tmp@^0.0.33: os-tmpdir "~1.0.2" tmpl@1.0.x: - version "1.0.4" - resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1" - integrity sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE= + version "1.0.5" + resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" + integrity sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw== to-fast-properties@^2.0.0: version "2.0.0" From eb87b3439b5fa24d2c4195bca9b79d200d7555a4 Mon Sep 17 00:00:00 2001 From: Samir Talwar Date: Tue, 28 Sep 2021 15:25:34 +0200 Subject: [PATCH 6/9] kvutils: Add the logging context for ledger state operations. (#11030) We do some logging in proprietary ledger drivers which would benefit from correct propagation of the context. CHANGELOG_BEGIN - [Integration Kit] We have added ``loggingContext`` as an implicit parameter to more _kvutils_ trait methods. Implementors may need to do the same in their trait implementations. This can make it easier to log with the appropriate context. CHANGELOG_END --- .../on/memory/InMemoryLedgerStateAccess.scala | 8 +- .../InMemoryLedgerStateOperations.scala | 10 ++- .../ledger/on/sql/SqlLedgerReaderWriter.scala | 22 +++-- .../SubmissionAggregatorWriteOperations.scala | 7 +- .../CombinedLedgerStateWriteOperations.scala | 10 ++- .../ledger/validator/CommitStrategy.scala | 3 +- .../ledger/validator/LedgerStateAccess.scala | 52 +++++++---- .../LogAppendingCommitStrategy.scala | 3 +- .../RawToDamlLedgerStateReaderAdapter.scala | 6 +- .../validator/SubmissionValidator.scala | 7 +- .../batch/BatchedSubmissionValidator.scala | 26 +++--- .../BatchedSubmissionValidatorFactory.scala | 6 +- .../caching/CachingCommitStrategy.scala | 3 +- .../caching/CachingStateReader.scala | 6 +- ...tyBasedPostExecutionConflictDetector.scala | 3 +- .../PostExecutionConflictDetector.scala | 5 +- .../preexecution/PostExecutionWriter.scala | 6 +- .../PreExecutingSubmissionValidator.scala | 13 +-- .../preexecution/RawPostExecutionWriter.scala | 6 +- .../LedgerStateOperationsReaderAdapter.scala | 6 +- .../validator/reading/StateReader.scala | 16 +++- .../ledger/validator/ArgumentMatchers.scala | 2 + .../daml/ledger/validator/TestHelper.scala | 3 +- .../preexecution/PreExecutionTestHelper.scala | 7 +- .../LogAppendingCommitStrategySpec.scala | 24 +++-- ...awToDamlLedgerStateReaderAdapterSpec.scala | 11 ++- .../validator/SubmissionValidatorSpec.scala | 87 +++++++++++++------ .../BatchedSubmissionValidatorSpec.scala | 33 ++++--- .../caching/CachingCommitStrategySpec.scala | 5 +- .../caching/CachingStateReaderSpec.scala | 18 ++-- .../PreExecutingValidatingCommitterSpec.scala | 28 +++--- .../kvutils/tools/BUILD.bazel | 2 + .../LogAppendingCommitStrategySupport.scala | 3 +- ...RawPreExecutingCommitStrategySupport.scala | 14 ++- .../WriteRecordingLedgerStateAccess.scala | 22 +++-- ...reExecutingCommitStrategySupportSpec.scala | 3 + .../CommitStrategySupport.scala | 3 +- .../integritycheck/IntegrityChecker.scala | 41 +++++---- 38 files changed, 363 insertions(+), 167 deletions(-) diff --git a/ledger/ledger-on-memory/src/main/scala/com/daml/ledger/on/memory/InMemoryLedgerStateAccess.scala b/ledger/ledger-on-memory/src/main/scala/com/daml/ledger/on/memory/InMemoryLedgerStateAccess.scala index a6b24cdf928d..c05aacaa06cb 100644 --- a/ledger/ledger-on-memory/src/main/scala/com/daml/ledger/on/memory/InMemoryLedgerStateAccess.scala +++ b/ledger/ledger-on-memory/src/main/scala/com/daml/ledger/on/memory/InMemoryLedgerStateAccess.scala @@ -8,14 +8,18 @@ import com.daml.ledger.validator.{ LedgerStateOperations, TimedLedgerStateOperations, } +import com.daml.logging.LoggingContext import com.daml.metrics.Metrics import scala.concurrent.{ExecutionContext, Future} final class InMemoryLedgerStateAccess(state: InMemoryState, metrics: Metrics) extends LedgerStateAccess[Index] { - override def inTransaction[T](body: LedgerStateOperations[Index] => Future[T])(implicit - executionContext: ExecutionContext + override def inTransaction[T]( + body: LedgerStateOperations[Index] => Future[T] + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, ): Future[T] = state.write { (log, state) => body(new TimedLedgerStateOperations(new InMemoryLedgerStateOperations(log, state), metrics)) diff --git a/ledger/ledger-on-memory/src/main/scala/com/daml/ledger/on/memory/InMemoryLedgerStateOperations.scala b/ledger/ledger-on-memory/src/main/scala/com/daml/ledger/on/memory/InMemoryLedgerStateOperations.scala index b03a838fdd49..472f02872798 100644 --- a/ledger/ledger-on-memory/src/main/scala/com/daml/ledger/on/memory/InMemoryLedgerStateOperations.scala +++ b/ledger/ledger-on-memory/src/main/scala/com/daml/ledger/on/memory/InMemoryLedgerStateOperations.scala @@ -9,6 +9,7 @@ import com.daml.ledger.on.memory.InMemoryState.MutableLog import com.daml.ledger.participant.state.kvutils.api.LedgerRecord import com.daml.ledger.participant.state.kvutils.{OffsetBuilder, Raw} import com.daml.ledger.validator.BatchingLedgerStateOperations +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -19,12 +20,15 @@ final class InMemoryLedgerStateOperations( override def readState( keys: Iterable[Raw.StateKey] - )(implicit executionContext: ExecutionContext): Future[Seq[Option[Raw.Envelope]]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[Option[Raw.Envelope]]] = Future.successful(keys.view.map(state.get).toSeq) override def writeState( keyValuePairs: Iterable[Raw.StateEntry] - )(implicit executionContext: ExecutionContext): Future[Unit] = { + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = { state ++= keyValuePairs Future.unit } @@ -32,7 +36,7 @@ final class InMemoryLedgerStateOperations( override def appendToLog( key: Raw.LogEntryId, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[Index] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Index] = Future.successful(appendEntry(log, LedgerRecord(_, key, value))) } diff --git a/ledger/ledger-on-sql/src/main/scala/com/daml/ledger/on/sql/SqlLedgerReaderWriter.scala b/ledger/ledger-on-sql/src/main/scala/com/daml/ledger/on/sql/SqlLedgerReaderWriter.scala index ea226e4400f3..9e64936f48b2 100644 --- a/ledger/ledger-on-sql/src/main/scala/com/daml/ledger/on/sql/SqlLedgerReaderWriter.scala +++ b/ledger/ledger-on-sql/src/main/scala/com/daml/ledger/on/sql/SqlLedgerReaderWriter.scala @@ -186,8 +186,11 @@ object SqlLedgerReaderWriter { private final class SqlLedgerStateAccess(database: Database, metrics: Metrics) extends LedgerStateAccess[Index] { - override def inTransaction[T](body: LedgerStateOperations[Index] => sc.Future[T])(implicit - executionContext: sc.ExecutionContext + override def inTransaction[T]( + body: LedgerStateOperations[Index] => sc.Future[T] + )(implicit + executionContext: sc.ExecutionContext, + loggingContext: LoggingContext, ): sc.Future[T] = database .inWriteTransaction("commit") { queries => @@ -200,18 +203,27 @@ object SqlLedgerReaderWriter { extends BatchingLedgerStateOperations[Index] { override def readState( keys: Iterable[Raw.StateKey] - )(implicit executionContext: sc.ExecutionContext): sc.Future[Seq[Option[Raw.Envelope]]] = + )(implicit + executionContext: sc.ExecutionContext, + loggingContext: LoggingContext, + ): sc.Future[Seq[Option[Raw.Envelope]]] = Future.fromTry(queries.selectStateValuesByKeys(keys)).removeExecutionContext override def writeState( keyValuePairs: Iterable[Raw.StateEntry] - )(implicit executionContext: sc.ExecutionContext): sc.Future[Unit] = + )(implicit + executionContext: sc.ExecutionContext, + loggingContext: LoggingContext, + ): sc.Future[Unit] = Future.fromTry(queries.updateState(keyValuePairs)).removeExecutionContext override def appendToLog( key: Raw.LogEntryId, value: Raw.Envelope, - )(implicit executionContext: sc.ExecutionContext): sc.Future[Index] = + )(implicit + executionContext: sc.ExecutionContext, + loggingContext: LoggingContext, + ): sc.Future[Index] = Future.fromTry(queries.insertRecordIntoLog(key, value)).removeExecutionContext } diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/participant/state/kvutils/export/SubmissionAggregatorWriteOperations.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/participant/state/kvutils/export/SubmissionAggregatorWriteOperations.scala index 0b51bcf166b5..b0f7ae80026a 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/participant/state/kvutils/export/SubmissionAggregatorWriteOperations.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/participant/state/kvutils/export/SubmissionAggregatorWriteOperations.scala @@ -5,6 +5,7 @@ package com.daml.ledger.participant.state.kvutils.export import com.daml.ledger.participant.state.kvutils.Raw import com.daml.ledger.validator.LedgerStateWriteOperations +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -14,14 +15,14 @@ final class SubmissionAggregatorWriteOperations(builder: SubmissionAggregator.Wr override def writeState( key: Raw.StateKey, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[Unit] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = Future { builder += key -> value } override def writeState( keyValuePairs: Iterable[Raw.StateEntry] - )(implicit executionContext: ExecutionContext): Future[Unit] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = Future { builder ++= keyValuePairs } @@ -29,7 +30,7 @@ final class SubmissionAggregatorWriteOperations(builder: SubmissionAggregator.Wr override def appendToLog( key: Raw.LogEntryId, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[Unit] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = Future { builder += key -> value } diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/CombinedLedgerStateWriteOperations.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/CombinedLedgerStateWriteOperations.scala index ef8b912bb7d3..1b4c7a49bbfd 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/CombinedLedgerStateWriteOperations.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/CombinedLedgerStateWriteOperations.scala @@ -4,6 +4,7 @@ package com.daml.ledger.validator import com.daml.ledger.participant.state.kvutils.Raw +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -16,18 +17,21 @@ final class CombinedLedgerStateWriteOperations[ALogResult, BLogResult, LogResult override def writeState( key: Raw.StateKey, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[Unit] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = inParallel(a.writeState(key, value), b.writeState(key, value)).map(_ => ()) override def writeState( keyValuePairs: Iterable[Raw.StateEntry] - )(implicit executionContext: ExecutionContext): Future[Unit] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = inParallel(a.writeState(keyValuePairs), b.writeState(keyValuePairs)).map(_ => ()) override def appendToLog( key: Raw.LogEntryId, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[LogResult] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[LogResult] = inParallel(a.appendToLog(key, value), b.appendToLog(key, value)).map { case (aResult, bResult) => combineLogResults(aResult, bResult) } diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/CommitStrategy.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/CommitStrategy.scala index e7b2ca007a1a..9b9b2e4623b9 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/CommitStrategy.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/CommitStrategy.scala @@ -11,6 +11,7 @@ import com.daml.ledger.participant.state.kvutils.DamlKvutils.{ } import com.daml.ledger.participant.state.kvutils.export.SubmissionAggregator import com.daml.lf.data.Ref +import com.daml.logging.LoggingContext import scala.concurrent.Future @@ -29,5 +30,5 @@ trait CommitStrategy[Result] { inputState: Map[DamlStateKey, Option[DamlStateValue]], outputState: Map[DamlStateKey, DamlStateValue], exporterWriteSet: Option[SubmissionAggregator.WriteSetBuilder] = None, - ): Future[Result] + )(implicit loggingContext: LoggingContext): Future[Result] } diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/LedgerStateAccess.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/LedgerStateAccess.scala index 2b96af2c26a9..24070cfe6243 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/LedgerStateAccess.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/LedgerStateAccess.scala @@ -5,6 +5,7 @@ package com.daml.ledger.validator import com.daml.dec.DirectExecutionContext import com.daml.ledger.participant.state.kvutils.Raw +import com.daml.logging.LoggingContext import com.daml.metrics.{Metrics, Timed} import scala.concurrent.{ExecutionContext, Future} @@ -23,7 +24,7 @@ trait LedgerStateAccess[+LogResult] { */ def inTransaction[T]( body: LedgerStateOperations[LogResult] => Future[T] - )(implicit executionContext: ExecutionContext): Future[T] + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[T] } /** Defines how the validator/committer can read from the backing store of the ledger. @@ -37,7 +38,10 @@ trait LedgerStateReadOperations { */ def readState( key: Raw.StateKey - )(implicit executionContext: ExecutionContext): Future[Option[Raw.Envelope]] + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Option[Raw.Envelope]] /** Reads values of a set of keys from the backing store. * @@ -46,7 +50,10 @@ trait LedgerStateReadOperations { */ def readState( keys: Iterable[Raw.StateKey] - )(implicit executionContext: ExecutionContext): Future[Seq[Option[Raw.Envelope]]] + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[Option[Raw.Envelope]]] } /** Defines how the validator/committer can write to the backing store of the ledger. @@ -60,13 +67,13 @@ trait LedgerStateWriteOperations[+LogResult] { def writeState( key: Raw.StateKey, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[Unit] + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] /** Writes a list of key-value pairs to the backing store. In case a key already exists its value is overwritten. */ def writeState( keyValuePairs: Iterable[Raw.StateEntry] - )(implicit executionContext: ExecutionContext): Future[Unit] + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] /** Writes a single log entry to the backing store. The implementation may return Future.failed in case the key * (i.e., the log entry ID) already exists. @@ -76,7 +83,7 @@ trait LedgerStateWriteOperations[+LogResult] { def appendToLog( key: Raw.LogEntryId, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[LogResult] + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[LogResult] } /** Defines how the validator/committer can access the backing store of the ledger. @@ -92,13 +99,16 @@ trait LedgerStateOperations[+LogResult] abstract class BatchingLedgerStateOperations[LogResult] extends LedgerStateOperations[LogResult] { override final def readState( key: Raw.StateKey - )(implicit executionContext: ExecutionContext): Future[Option[Raw.Envelope]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Option[Raw.Envelope]] = readState(Seq(key)).map(_.head)(DirectExecutionContext) override final def writeState( key: Raw.StateKey, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[Unit] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = writeState(Seq(key -> value)) } @@ -109,12 +119,15 @@ abstract class NonBatchingLedgerStateOperations[LogResult] extends LedgerStateOperations[LogResult] { override final def readState( keys: Iterable[Raw.StateKey] - )(implicit executionContext: ExecutionContext): Future[Seq[Option[Raw.Envelope]]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[Option[Raw.Envelope]]] = Future.sequence(keys.map(readState)).map(_.toSeq) override final def writeState( keyValuePairs: Iterable[Raw.StateEntry] - )(implicit executionContext: ExecutionContext): Future[Unit] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = Future .sequence(keyValuePairs.map { case (key, value) => writeState(key, value) @@ -129,28 +142,37 @@ final class TimedLedgerStateOperations[LogResult]( override def readState( key: Raw.StateKey - )(implicit executionContext: ExecutionContext): Future[Option[Raw.Envelope]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Option[Raw.Envelope]] = Timed.future(metrics.daml.ledger.state.read, delegate.readState(key)) override def readState( keys: Iterable[Raw.StateKey] - )(implicit executionContext: ExecutionContext): Future[Seq[Option[Raw.Envelope]]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[Option[Raw.Envelope]]] = Timed.future(metrics.daml.ledger.state.read, delegate.readState(keys)) override def writeState( key: Raw.StateKey, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[Unit] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = Timed.future(metrics.daml.ledger.state.write, delegate.writeState(key, value)) override def writeState( keyValuePairs: Iterable[Raw.StateEntry] - )(implicit executionContext: ExecutionContext): Future[Unit] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = Timed.future(metrics.daml.ledger.state.write, delegate.writeState(keyValuePairs)) override def appendToLog( key: Raw.LogEntryId, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[LogResult] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[LogResult] = Timed.future(metrics.daml.ledger.log.append, delegate.appendToLog(key, value)) } diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/LogAppendingCommitStrategy.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/LogAppendingCommitStrategy.scala index 2d8dea4e1c61..b0dd1eca74d2 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/LogAppendingCommitStrategy.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/LogAppendingCommitStrategy.scala @@ -12,6 +12,7 @@ import com.daml.ledger.participant.state.kvutils.DamlKvutils.{ import com.daml.ledger.participant.state.kvutils.export.SubmissionAggregator import com.daml.ledger.participant.state.kvutils.{Envelope, Raw} import com.daml.lf.data.Ref +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -30,7 +31,7 @@ class LogAppendingCommitStrategy[Index]( inputState: Map[DamlStateKey, Option[DamlStateValue]], outputState: Map[DamlStateKey, DamlStateValue], writeSetBuilder: Option[SubmissionAggregator.WriteSetBuilder] = None, - ): Future[Index] = { + )(implicit loggingContext: LoggingContext): Future[Index] = { val rawLogEntryId = Raw.LogEntryId(entryId) for { (serializedKeyValuePairs, envelopedLogEntry) <- inParallel( diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/RawToDamlLedgerStateReaderAdapter.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/RawToDamlLedgerStateReaderAdapter.scala index 9513b06565d5..1cd2732be406 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/RawToDamlLedgerStateReaderAdapter.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/RawToDamlLedgerStateReaderAdapter.scala @@ -6,6 +6,7 @@ package com.daml.ledger.validator import com.daml.ledger.participant.state.kvutils.DamlKvutils.{DamlStateKey, DamlStateValue} import com.daml.ledger.participant.state.kvutils.{Envelope, Raw} import com.daml.ledger.validator.reading.{DamlLedgerStateReader, LedgerStateReader} +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -18,7 +19,10 @@ final class RawToDamlLedgerStateReaderAdapter( override def read( keys: Iterable[DamlStateKey] - )(implicit executionContext: ExecutionContext): Future[Seq[Option[DamlStateValue]]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[Option[DamlStateValue]]] = ledgerStateReader .read(keys.map(keySerializationStrategy.serializeStateKey)) .map(_.map(_.map(deserializeDamlStateValue))) diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/SubmissionValidator.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/SubmissionValidator.scala index aa905bd013c8..3d21a5a346c6 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/SubmissionValidator.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/SubmissionValidator.scala @@ -121,7 +121,10 @@ class SubmissionValidator[LogResult] private[validator] ( ignored: Any, logEntryAndState: LogEntryAndState, stateOperations: LedgerStateOperations[LogResult], - )(implicit executionContext: ExecutionContext): Future[LogResult] = { + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[LogResult] = { val (rawLogEntry, rawStateUpdates) = serializeProcessedSubmission(logEntryAndState) val eventualLogResult = stateOperations.appendToLog(Raw.LogEntryId(logEntryId), rawLogEntry) val eventualStateResult = @@ -266,7 +269,7 @@ class SubmissionValidator[LogResult] private[validator] ( extends LedgerStateAccess[LogResult] { override def inTransaction[T]( body: LedgerStateOperations[LogResult] => Future[T] - )(implicit executionContext: ExecutionContext): Future[T] = { + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[T] = { // This is necessary to ensure we capture successful and failed acquisitions separately. // These need to be measured separately as they may have very different characteristics. val acquisitionWasRecorded = new AtomicBoolean(false) diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidator.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidator.scala index e028ba911845..c0ce35c6619a 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidator.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidator.scala @@ -319,7 +319,10 @@ class BatchedSubmissionValidator[CommitResult] private[validator] ( private def fetchSubmissionInputs( correlatedSubmission: CorrelatedSubmission, ledgerStateReader: DamlLedgerStateReader, - )(implicit executionContext: ExecutionContext): Future[FetchedInput] = { + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[FetchedInput] = { val inputKeys = correlatedSubmission.submission.getInputDamlStateList.asScala withSubmissionLoggingContext(correlatedSubmission) { _ => Timed.timedAndTrackedFuture( @@ -407,23 +410,22 @@ class BatchedSubmissionValidator[CommitResult] private[validator] ( logEntryAndState: LogEntryAndState, commitStrategy: CommitStrategy[CommitResult], exporterWriteSet: SubmissionAggregator.WriteSetBuilder, - )(implicit executionContext: ExecutionContext): Future[Unit] = { + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = { val (logEntry, outputState) = logEntryAndState withSubmissionLoggingContext(correlatedSubmission) { _ => Timed .timedAndTrackedFuture( metrics.commit, metrics.commitRunning, - commitStrategy - .commit( - participantId, - correlatedSubmission.correlationId, - correlatedSubmission.logEntryId, - logEntry, - inputState, - outputState, - Some(exporterWriteSet), - ), + commitStrategy.commit( + participantId, + correlatedSubmission.correlationId, + correlatedSubmission.logEntryId, + logEntry, + inputState, + outputState, + Some(exporterWriteSet), + ), ) .map(_ => ()) } diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidatorFactory.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidatorFactory.scala index 44a6c40f6a9c..ee9ef446a348 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidatorFactory.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidatorFactory.scala @@ -20,6 +20,7 @@ import com.daml.ledger.validator.{ LogAppendingCommitStrategy, StateKeySerializationStrategy, } +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -38,7 +39,10 @@ object BatchedSubmissionValidatorFactory { extends LedgerStateReader { override def read( keys: Iterable[Raw.StateKey] - )(implicit executionContext: ExecutionContext): Future[Seq[Option[Raw.Envelope]]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[Option[Raw.Envelope]]] = delegate.readState(keys) } diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/caching/CachingCommitStrategy.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/caching/CachingCommitStrategy.scala index e7762d53bd41..6a0a6f5b1ecc 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/caching/CachingCommitStrategy.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/caching/CachingCommitStrategy.scala @@ -14,6 +14,7 @@ import com.daml.ledger.validator.{ StateKeySerializationStrategy, } import com.daml.lf.data.Ref +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -31,7 +32,7 @@ final class CachingCommitStrategy[Result]( inputState: Map[DamlStateKey, Option[DamlStateValue]], outputState: Map[DamlStateKey, DamlStateValue], exporterWriteSet: Option[SubmissionAggregator.WriteSetBuilder], - ): Future[Result] = + )(implicit loggingContext: LoggingContext): Future[Result] = for { _ <- Future { outputState.view.filter { case (key, _) => shouldCache(key) }.foreach { case (key, value) => diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/caching/CachingStateReader.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/caching/CachingStateReader.scala index 37ae0994ddfc..a4745c0bbb74 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/caching/CachingStateReader.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/caching/CachingStateReader.scala @@ -5,6 +5,7 @@ package com.daml.ledger.validator.caching import com.daml.caching.Cache import com.daml.ledger.validator.reading.StateReader +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -20,7 +21,10 @@ final class CachingStateReader[Key, Value]( ) extends StateReader[Key, Value] { override def read( keys: Iterable[Key] - )(implicit executionContext: ExecutionContext): Future[Seq[Value]] = { + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[Value]] = { @SuppressWarnings(Array("org.wartremover.warts.Any")) // Required to make `.view` work. val cachedValues = keys.view .map(key => key -> cache.getIfPresent(key)) diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/EqualityBasedPostExecutionConflictDetector.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/EqualityBasedPostExecutionConflictDetector.scala index e7727a6839a2..339d622f5aed 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/EqualityBasedPostExecutionConflictDetector.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/EqualityBasedPostExecutionConflictDetector.scala @@ -4,6 +4,7 @@ package com.daml.ledger.validator.preexecution import com.daml.ledger.validator.reading.StateReader +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -12,7 +13,7 @@ final class EqualityBasedPostExecutionConflictDetector[StateKey, StateValue] override def detectConflicts( preExecutionOutput: PreExecutionOutput[Map[StateKey, StateValue], Any], reader: StateReader[StateKey, StateValue], - )(implicit executionContext: ExecutionContext): Future[Unit] = { + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = { val (keys, preExecutionValues) = preExecutionOutput.readSet.unzip reader.read(keys).map { currentValues => if (preExecutionValues != currentValues) { diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PostExecutionConflictDetector.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PostExecutionConflictDetector.scala index 4f0cccc5886b..9a958636b1b9 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PostExecutionConflictDetector.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PostExecutionConflictDetector.scala @@ -4,6 +4,7 @@ package com.daml.ledger.validator.preexecution import com.daml.ledger.validator.reading.StateReader +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -26,7 +27,7 @@ trait PostExecutionConflictDetector[StateKey, StateValue, -ReadSet, -WriteSet] { def detectConflicts( preExecutionOutput: PreExecutionOutput[ReadSet, WriteSet], reader: StateReader[StateKey, StateValue], - )(implicit executionContext: ExecutionContext): Future[Unit] + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] /** Transforms the reader to widen the state value, allowing it to handle any value that can be * converted to `StateValue`. @@ -44,7 +45,7 @@ trait PostExecutionConflictDetector[StateKey, StateValue, -ReadSet, -WriteSet] { override def detectConflicts( preExecutionOutput: PreExecutionOutput[ReadSet, WriteSet], reader: StateReader[StateKey, NewStateValue], - )(implicit executionContext: ExecutionContext): Future[Unit] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = self.detectConflicts(preExecutionOutput, reader.mapValues(transformValue)) } } diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PostExecutionWriter.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PostExecutionWriter.scala index 2d4346c107d0..2d730e87b403 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PostExecutionWriter.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PostExecutionWriter.scala @@ -5,6 +5,7 @@ package com.daml.ledger.validator.preexecution import com.daml.ledger.participant.state.v2.SubmissionResult import com.daml.ledger.validator.LedgerStateWriteOperations +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -12,5 +13,8 @@ trait PostExecutionWriter[WriteSet] { def write[LogResult]( writeSet: WriteSet, operations: LedgerStateWriteOperations[LogResult], - )(implicit executionContext: ExecutionContext): Future[SubmissionResult] + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[SubmissionResult] } diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PreExecutingSubmissionValidator.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PreExecutingSubmissionValidator.scala index 6b70cf8d7646..a4d9ee2cbe66 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PreExecutingSubmissionValidator.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PreExecutingSubmissionValidator.scala @@ -24,10 +24,10 @@ import scala.jdk.CollectionConverters._ /** Validator for pre-executing submissions. * - * @param committer Generates the pre-execution result from the submission. - * @param commitStrategy The strategy used to generate the data committed to the ledger. - * @param logEntryIdComputationStrategy The strategy used to generate the log entry id. - * @param metrics Records metrics. + * @param committer Generates the pre-execution result from the submission. + * @param commitStrategy The strategy used to generate the data committed to the ledger. + * @param logEntryIdComputationStrategy The strategy used to generate the log entry id. + * @param metrics Records metrics. * @tparam StateValue The type of the state persisted to the ledger. This must implement * [[HasDamlStateValue]]. * @tparam ReadSet The type of the read set generated by the `commitStrategy`. @@ -124,7 +124,10 @@ class PreExecutingSubmissionValidator[StateValue, ReadSet, WriteSet]( private def fetchSubmissionInputs( submission: DamlSubmission, ledgerStateReader: StateReader[DamlStateKey, StateValue], - )(implicit executionContext: ExecutionContext): Future[Map[DamlStateKey, StateValue]] = { + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Map[DamlStateKey, StateValue]] = { val inputKeys = submission.getInputDamlStateList.asScala Timed.timedAndTrackedFuture( metrics.daml.kvutils.submission.validator.fetchInputs, diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/RawPostExecutionWriter.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/RawPostExecutionWriter.scala index d45a39b5d373..b20b958a6652 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/RawPostExecutionWriter.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/RawPostExecutionWriter.scala @@ -5,6 +5,7 @@ package com.daml.ledger.validator.preexecution import com.daml.ledger.participant.state.v2.SubmissionResult import com.daml.ledger.validator.LedgerStateWriteOperations +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -13,7 +14,10 @@ final class RawPostExecutionWriter extends PostExecutionWriter[RawKeyValuePairsW override def write[LogResult]( writeSet: RawKeyValuePairsWithLogEntry, operations: LedgerStateWriteOperations[LogResult], - )(implicit executionContext: ExecutionContext): Future[SubmissionResult] = { + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[SubmissionResult] = { for { _ <- operations.writeState(writeSet.state) _ <- operations.appendToLog(writeSet.logEntryKey, writeSet.logEntryValue) diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/reading/LedgerStateOperationsReaderAdapter.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/reading/LedgerStateOperationsReaderAdapter.scala index c50b14831dc1..228751cfa7d3 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/reading/LedgerStateOperationsReaderAdapter.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/reading/LedgerStateOperationsReaderAdapter.scala @@ -5,6 +5,7 @@ package com.daml.ledger.validator import com.daml.ledger.participant.state.kvutils.Raw import com.daml.ledger.validator.reading.StateReader +import com.daml.logging.LoggingContext import scala.concurrent.{ExecutionContext, Future} @@ -13,6 +14,9 @@ final class LedgerStateOperationsReaderAdapter[LogResult]( ) extends StateReader[Raw.StateKey, Option[Raw.Envelope]] { override def read( keys: Iterable[Raw.StateKey] - )(implicit executionContext: ExecutionContext): Future[Seq[Option[Raw.Envelope]]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[Option[Raw.Envelope]]] = operations.readState(keys) } diff --git a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/reading/StateReader.scala b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/reading/StateReader.scala index 3631d616491d..70e87dc49a3c 100644 --- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/reading/StateReader.scala +++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/reading/StateReader.scala @@ -3,6 +3,8 @@ package com.daml.ledger.validator.reading +import com.daml.logging.LoggingContext + import scala.concurrent.{ExecutionContext, Future} /** Generic interface for reading from the ledger. @@ -21,7 +23,9 @@ trait StateReader[-Key, +Value] { * @param keys list of keys to look up * @return values corresponding to the requested keys, in the same order as requested */ - def read(keys: Iterable[Key])(implicit executionContext: ExecutionContext): Future[Seq[Value]] + def read( + keys: Iterable[Key] + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Seq[Value]] /** Create a new StateReader that transforms the keys before reading. * @@ -36,7 +40,10 @@ trait StateReader[-Key, +Value] { new StateReader[NewKey, Value] { override def read( keys: Iterable[NewKey] - )(implicit executionContext: ExecutionContext): Future[Seq[Value]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[Value]] = self.read(keys.map(f)) } @@ -50,7 +57,10 @@ trait StateReader[-Key, +Value] { new StateReader[Key, NewValue] { override def read( keys: Iterable[Key] - )(implicit executionContext: ExecutionContext): Future[Seq[NewValue]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[NewValue]] = self.read(keys).map(_.map(f)) } } diff --git a/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/ArgumentMatchers.scala b/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/ArgumentMatchers.scala index ed48cfe915f5..ee6bb286dad3 100644 --- a/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/ArgumentMatchers.scala +++ b/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/ArgumentMatchers.scala @@ -3,6 +3,7 @@ package com.daml.ledger.validator +import com.daml.logging.LoggingContext import org.mockito.{ArgumentMatcher, ArgumentMatchersSugar} import scala.concurrent.ExecutionContext @@ -12,6 +13,7 @@ trait ArgumentMatchers { import ArgumentMatchersSugar._ def anyExecutionContext: ExecutionContext = any[ExecutionContext] + def anyLoggingContext: LoggingContext = any[LoggingContext] def iterableOf[T](size: Int): Iterable[T] = argThat[Iterable[T]](new ArgumentMatcher[Iterable[T]] { diff --git a/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/TestHelper.scala b/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/TestHelper.scala index 76aa9373ddb8..f443525f12db 100644 --- a/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/TestHelper.scala +++ b/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/TestHelper.scala @@ -10,6 +10,7 @@ import com.daml.ledger.participant.state.kvutils.Raw import com.daml.ledger.participant.state.kvutils.wire.DamlSubmission import com.daml.lf.data.Ref import com.daml.lf.value.ValueOuterClass.Identifier +import com.daml.logging.LoggingContext import com.google.protobuf.{ByteString, Empty} import scala.concurrent.{ExecutionContext, Future} @@ -93,7 +94,7 @@ private[ledger] object TestHelper { extends LedgerStateAccess[LogResult] { override def inTransaction[T]( body: LedgerStateOperations[LogResult] => Future[T] - )(implicit executionContext: ExecutionContext): Future[T] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[T] = body(mockStateOperations) } } diff --git a/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/preexecution/PreExecutionTestHelper.scala b/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/preexecution/PreExecutionTestHelper.scala index a7df469b6690..4a0f5f243172 100644 --- a/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/preexecution/PreExecutionTestHelper.scala +++ b/ledger/participant-state/kvutils/src/test/lib/scala/com/daml/ledger/validator/preexecution/PreExecutionTestHelper.scala @@ -1,10 +1,12 @@ // Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. // SPDX-License-Identifier: Apache-2.0 + package com.daml.ledger.validator.preexecution import com.daml.ledger.participant.state.kvutils.DamlKvutils.{DamlStateKey, DamlStateValue} import com.daml.ledger.validator.HasDamlStateValue import com.daml.ledger.validator.reading.StateReader +import com.daml.logging.LoggingContext import scala.collection.compat._ import scala.concurrent.{ExecutionContext, Future} @@ -24,7 +26,10 @@ object PreExecutionTestHelper { new StateReader[DamlStateKey, TestValue] { override def read( keys: Iterable[DamlStateKey] - )(implicit executionContext: ExecutionContext): Future[Seq[TestValue]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[TestValue]] = Future.successful(keys.view.map(wrappedInputState).toVector) } } diff --git a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/LogAppendingCommitStrategySpec.scala b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/LogAppendingCommitStrategySpec.scala index 197035d77254..04a6dd1e6180 100644 --- a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/LogAppendingCommitStrategySpec.scala +++ b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/LogAppendingCommitStrategySpec.scala @@ -5,9 +5,10 @@ package com.daml.ledger.validator import com.daml.ledger.participant.state.kvutils.DamlKvutils.{DamlStateKey, DamlStateValue} import com.daml.ledger.participant.state.kvutils.{Envelope, Raw} -import com.daml.ledger.validator.ArgumentMatchers.anyExecutionContext +import com.daml.ledger.validator.ArgumentMatchers.{anyExecutionContext, anyLoggingContext} import com.daml.ledger.validator.LogAppendingCommitStrategySpec._ import com.daml.ledger.validator.TestHelper._ +import com.daml.logging.LoggingContext import com.google.protobuf.ByteString import org.mockito.{ArgumentMatchersSugar, MockitoSugar} import org.scalatest.matchers.should.Matchers @@ -20,6 +21,8 @@ final class LogAppendingCommitStrategySpec with Matchers with MockitoSugar with ArgumentMatchersSugar { + private implicit val loggingContext: LoggingContext = LoggingContext.ForTesting + "commit" should { "return index from appendToLog" in { val mockLedgerStateOperations = mock[LedgerStateOperations[Long]] @@ -28,7 +31,7 @@ final class LogAppendingCommitStrategySpec mockLedgerStateOperations.appendToLog( any[Raw.LogEntryId], any[Raw.Envelope], - )(anyExecutionContext) + )(anyExecutionContext, anyLoggingContext) ).thenReturn(Future.successful(expectedIndex)) val instance = new LogAppendingCommitStrategy[Long]( @@ -40,9 +43,12 @@ final class LogAppendingCommitStrategySpec .commit(aParticipantId, "a correlation ID", aLogEntryId(), aLogEntry, Map.empty, Map.empty) .map { actualIndex => verify(mockLedgerStateOperations, times(1)) - .appendToLog(any[Raw.LogEntryId], any[Raw.Envelope])(anyExecutionContext) + .appendToLog(any[Raw.LogEntryId], any[Raw.Envelope])( + anyExecutionContext, + anyLoggingContext, + ) verify(mockLedgerStateOperations, times(0)) - .writeState(any[Iterable[Raw.StateEntry]])(anyExecutionContext) + .writeState(any[Iterable[Raw.StateEntry]])(anyExecutionContext, anyLoggingContext) actualIndex should be(expectedIndex) } } @@ -50,14 +56,14 @@ final class LogAppendingCommitStrategySpec "write keys serialized according to strategy" in { val mockLedgerStateOperations = mock[LedgerStateOperations[Long]] when( - mockLedgerStateOperations.writeState(any[Iterable[Raw.StateEntry]])(anyExecutionContext) - ) - .thenReturn(Future.unit) + mockLedgerStateOperations + .writeState(any[Iterable[Raw.StateEntry]])(anyExecutionContext, anyLoggingContext) + ).thenReturn(Future.unit) when( mockLedgerStateOperations.appendToLog( any[Raw.LogEntryId], any[Raw.Envelope], - )(anyExecutionContext) + )(anyExecutionContext, anyLoggingContext) ).thenReturn(Future.successful(0L)) val mockStateKeySerializationStrategy = mock[StateKeySerializationStrategy] val expectedStateKey = Raw.StateKey(ByteString.copyFromUtf8("some key")) @@ -82,7 +88,7 @@ final class LogAppendingCommitStrategySpec .map { _: Long => verify(mockStateKeySerializationStrategy, times(1)).serializeStateKey(aStateKey) verify(mockLedgerStateOperations, times(1)) - .writeState(eqTo(expectedOutputStateBytes))(anyExecutionContext) + .writeState(eqTo(expectedOutputStateBytes))(anyExecutionContext, anyLoggingContext) succeed } } diff --git a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/RawToDamlLedgerStateReaderAdapterSpec.scala b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/RawToDamlLedgerStateReaderAdapterSpec.scala index c3805b82e227..26678842a9a7 100644 --- a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/RawToDamlLedgerStateReaderAdapterSpec.scala +++ b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/RawToDamlLedgerStateReaderAdapterSpec.scala @@ -9,10 +9,11 @@ import com.daml.ledger.participant.state.kvutils.DamlKvutils.{ DamlStateValue, } import com.daml.ledger.participant.state.kvutils.{Envelope, Raw} -import com.daml.ledger.validator.ArgumentMatchers.anyExecutionContext +import com.daml.ledger.validator.ArgumentMatchers.{anyExecutionContext, anyLoggingContext} import com.daml.ledger.validator.RawToDamlLedgerStateReaderAdapterSpec._ import com.daml.ledger.validator.TestHelper.{anInvalidEnvelope, makePartySubmission} import com.daml.ledger.validator.reading.LedgerStateReader +import com.daml.logging.LoggingContext import org.mockito.{ArgumentMatchersSugar, MockitoSugar} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AsyncWordSpec @@ -24,6 +25,8 @@ class RawToDamlLedgerStateReaderAdapterSpec with Matchers with MockitoSugar with ArgumentMatchersSugar { + private implicit val loggingContext: LoggingContext = LoggingContext.ForTesting + "readState" should { "read the right key and deserialize it" in { val expectedKey = DefaultStateKeySerializationStrategy.serializeStateKey(aDamlStateKey()) @@ -31,7 +34,7 @@ class RawToDamlLedgerStateReaderAdapterSpec .setParty(DamlPartyAllocation.newBuilder.setDisplayName("aParty")) .build val mockReader = mock[LedgerStateReader] - when(mockReader.read(any[Seq[Raw.StateKey]])(anyExecutionContext)) + when(mockReader.read(any[Seq[Raw.StateKey]])(anyExecutionContext, anyLoggingContext)) .thenReturn(Future.successful(Seq(Some(Envelope.enclose(expectedValue))))) val instance = new RawToDamlLedgerStateReaderAdapter(mockReader, DefaultStateKeySerializationStrategy) @@ -44,7 +47,7 @@ class RawToDamlLedgerStateReaderAdapterSpec "throw in case of an invalid envelope returned from underlying reader" in { val mockReader = mock[LedgerStateReader] - when(mockReader.read(any[Seq[Raw.StateKey]])(anyExecutionContext)) + when(mockReader.read(any[Seq[Raw.StateKey]])(anyExecutionContext, anyLoggingContext)) .thenReturn(Future.successful(Seq(Some(anInvalidEnvelope)))) val instance = new RawToDamlLedgerStateReaderAdapter(mockReader, DefaultStateKeySerializationStrategy) @@ -58,7 +61,7 @@ class RawToDamlLedgerStateReaderAdapterSpec "throw in case an enveloped value other than a DamlStateValue is returned from underlying reader" in { val notADamlStateValue = makePartySubmission("aParty") val mockReader = mock[LedgerStateReader] - when(mockReader.read(any[Seq[Raw.StateKey]])(anyExecutionContext)) + when(mockReader.read(any[Seq[Raw.StateKey]])(anyExecutionContext, anyLoggingContext)) .thenReturn(Future.successful(Seq(Some(Envelope.enclose(notADamlStateValue))))) val instance = new RawToDamlLedgerStateReaderAdapter(mockReader, DefaultStateKeySerializationStrategy) diff --git a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/SubmissionValidatorSpec.scala b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/SubmissionValidatorSpec.scala index a91c09b1fa1b..7d40c204d538 100644 --- a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/SubmissionValidatorSpec.scala +++ b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/SubmissionValidatorSpec.scala @@ -10,7 +10,7 @@ import com.daml.caching.Cache import com.daml.ledger.participant.state.kvutils.DamlKvutils._ import com.daml.ledger.participant.state.kvutils.wire.{DamlSubmission, DamlSubmissionBatch} import com.daml.ledger.participant.state.kvutils.{Envelope, KeyValueCommitting, Raw} -import com.daml.ledger.validator.ArgumentMatchers.anyExecutionContext +import com.daml.ledger.validator.ArgumentMatchers.{anyExecutionContext, anyLoggingContext} import com.daml.ledger.validator.SubmissionValidatorSpec._ import com.daml.ledger.validator.TestHelper.{ FakeStateAccess, @@ -41,8 +41,11 @@ class SubmissionValidatorSpec "validate" should { "return success in case of no errors during processing of submission" in { val mockStateOperations = mock[LedgerStateOperations[Unit]] - when(mockStateOperations.readState(any[Iterable[Raw.StateKey]])(anyExecutionContext)) - .thenReturn(Future.successful(Seq(Some(aStateValue())))) + when( + mockStateOperations.readState( + any[Iterable[Raw.StateKey]] + )(anyExecutionContext, anyLoggingContext) + ).thenReturn(Future.successful(Seq(Some(aStateValue())))) val metrics = new Metrics(new MetricRegistry) val instance = new SubmissionValidator( ledgerStateAccess = new FakeStateAccess(mockStateOperations), @@ -63,8 +66,11 @@ class SubmissionValidatorSpec "signal missing input in case state cannot be retrieved" in { val mockStateOperations = mock[LedgerStateOperations[Unit]] - when(mockStateOperations.readState(any[Iterable[Raw.StateKey]])(anyExecutionContext)) - .thenReturn(Future.successful(Seq(None))) + when( + mockStateOperations.readState( + any[Iterable[Raw.StateKey]] + )(anyExecutionContext, anyLoggingContext) + ).thenReturn(Future.successful(Seq(None))) val metrics = new Metrics(new MetricRegistry) val instance = new SubmissionValidator( ledgerStateAccess = new FakeStateAccess(mockStateOperations), @@ -110,8 +116,11 @@ class SubmissionValidatorSpec "return invalid submission in case exception is thrown during processing of submission" in { val mockStateOperations = mock[BatchingLedgerStateOperations[Unit]] - when(mockStateOperations.readState(any[Iterable[Raw.StateKey]])(anyExecutionContext)) - .thenReturn(Future.successful(Seq(Some(aStateValue())))) + when( + mockStateOperations.readState( + any[Iterable[Raw.StateKey]] + )(anyExecutionContext, anyLoggingContext) + ).thenReturn(Future.successful(Seq(Some(aStateValue())))) val failingProcessSubmission: SubmissionValidator.ProcessSubmission = (_, _, _, _, _) => throw new IllegalArgumentException("Validation failed") @@ -136,14 +145,18 @@ class SubmissionValidatorSpec "write marshalled log entry to ledger" in { val mockStateOperations = mock[LedgerStateOperations[Int]] val expectedLogResult: Int = 3 - when(mockStateOperations.readState(any[Iterable[Raw.StateKey]])(anyExecutionContext)) - .thenReturn(Future.successful(Seq(Some(aStateValue())))) + when( + mockStateOperations.readState( + any[Iterable[Raw.StateKey]] + )(anyExecutionContext, anyLoggingContext) + ).thenReturn(Future.successful(Seq(Some(aStateValue())))) val logEntryIdCaptor = ArgCaptor[Raw.LogEntryId] val logEntryValueCaptor = ArgCaptor[Raw.Envelope] when( - mockStateOperations.appendToLog(logEntryIdCaptor.capture, logEntryValueCaptor.capture)( - anyExecutionContext - ) + mockStateOperations.appendToLog( + logEntryIdCaptor.capture, + logEntryValueCaptor.capture, + )(anyExecutionContext, anyLoggingContext) ).thenReturn(Future.successful(expectedLogResult)) val expectedLogEntryId = aLogEntryId() val mockLogEntryIdGenerator = mock[LogEntryIdAllocator] @@ -165,7 +178,7 @@ class SubmissionValidatorSpec actualLogResult should be(expectedLogResult) verify(mockLogEntryIdGenerator, times(1)).allocate() verify(mockStateOperations, times(0)) - .writeState(any[Iterable[Raw.StateEntry]])(anyExecutionContext) + .writeState(any[Iterable[Raw.StateEntry]])(anyExecutionContext, anyLoggingContext) logEntryValueCaptor.values should have size 1 logEntryIdCaptor.values should be(List(Raw.LogEntryId(expectedLogEntryId))) } @@ -175,17 +188,23 @@ class SubmissionValidatorSpec "write marshalled key-value pairs to ledger" in { val mockStateOperations = mock[LedgerStateOperations[Int]] val expectedLogResult: Int = 7 - when(mockStateOperations.readState(any[Iterable[Raw.StateKey]])(anyExecutionContext)) - .thenReturn(Future.successful(Seq(Some(aStateValue())))) + when( + mockStateOperations.readState( + any[Iterable[Raw.StateKey]] + )(anyExecutionContext, anyLoggingContext) + ).thenReturn(Future.successful(Seq(Some(aStateValue())))) val writtenKeyValuesCaptor = ArgCaptor[Seq[Raw.StateEntry]] - when(mockStateOperations.writeState(writtenKeyValuesCaptor.capture)(anyExecutionContext)) - .thenReturn(Future.unit) + when( + mockStateOperations.writeState( + writtenKeyValuesCaptor.capture + )(anyExecutionContext, anyLoggingContext) + ).thenReturn(Future.unit) val logEntryCaptor = ArgCaptor[Raw.Envelope] when( mockStateOperations.appendToLog( any[Raw.LogEntryId], logEntryCaptor.capture, - )(anyExecutionContext) + )(anyExecutionContext, anyLoggingContext) ).thenReturn(Future.successful(expectedLogResult)) val logEntryAndStateResult = (aLogEntry, someStateUpdates) val instance = new SubmissionValidator( @@ -215,17 +234,23 @@ class SubmissionValidatorSpec "support batch with single submission" in { val mockStateOperations = mock[LedgerStateOperations[Int]] val expectedLogResult: Int = 7 - when(mockStateOperations.readState(any[Iterable[Raw.StateKey]])(anyExecutionContext)) - .thenReturn(Future.successful(Seq(Some(aStateValue())))) + when( + mockStateOperations.readState( + any[Iterable[Raw.StateKey]] + )(anyExecutionContext, anyLoggingContext) + ).thenReturn(Future.successful(Seq(Some(aStateValue())))) val writtenKeyValuesCaptor = ArgCaptor[Seq[Raw.StateEntry]] - when(mockStateOperations.writeState(writtenKeyValuesCaptor.capture)(anyExecutionContext)) - .thenReturn(Future.unit) + when( + mockStateOperations.writeState( + writtenKeyValuesCaptor.capture + )(anyExecutionContext, anyLoggingContext) + ).thenReturn(Future.unit) val logEntryCaptor = ArgCaptor[Raw.Envelope] when( mockStateOperations.appendToLog( any[Raw.LogEntryId], logEntryCaptor.capture, - )(anyExecutionContext) + )(anyExecutionContext, anyLoggingContext) ).thenReturn(Future.successful(expectedLogResult)) val logEntryAndStateResult = (aLogEntry, someStateUpdates) val instance = new SubmissionValidator( @@ -299,15 +324,21 @@ class SubmissionValidatorSpec "return invalid submission if state cannot be written" in { val mockStateOperations = mock[LedgerStateOperations[Int]] - when(mockStateOperations.writeState(any[Iterable[Raw.StateEntry]])(anyExecutionContext)) - .thenThrow(new IllegalArgumentException("Write error")) - when(mockStateOperations.readState(any[Iterable[Raw.StateKey]])(anyExecutionContext)) - .thenReturn(Future.successful(Seq(Some(aStateValue())))) + when( + mockStateOperations.writeState( + any[Iterable[Raw.StateEntry]] + )(anyExecutionContext, anyLoggingContext) + ).thenThrow(new IllegalArgumentException("Write error")) + when( + mockStateOperations.readState( + any[Iterable[Raw.StateKey]] + )(anyExecutionContext, anyLoggingContext) + ).thenReturn(Future.successful(Seq(Some(aStateValue())))) when( mockStateOperations.appendToLog( any[Raw.LogEntryId], any[Raw.Envelope], - )(anyExecutionContext) + )(anyExecutionContext, anyLoggingContext) ).thenReturn(Future.successful(99)) val logEntryAndStateResult = (aLogEntry, someStateUpdates) val instance = new SubmissionValidator( diff --git a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidatorSpec.scala b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidatorSpec.scala index 9364b059c7ea..5e32c007cbec 100644 --- a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidatorSpec.scala +++ b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/batch/BatchedSubmissionValidatorSpec.scala @@ -8,14 +8,18 @@ import java.time.Clock import com.codahale.metrics.MetricRegistry import com.daml.ledger.api.testing.utils.AkkaBeforeAndAfterAll import com.daml.ledger.participant.state.kvutils.DamlKvutils._ -import com.daml.ledger.participant.state.kvutils.wire.DamlSubmissionBatch.CorrelatedSubmission -import com.daml.ledger.participant.state.kvutils.wire._ import com.daml.ledger.participant.state.kvutils.export.{ NoOpLedgerDataExporter, SubmissionAggregator, } +import com.daml.ledger.participant.state.kvutils.wire.DamlSubmissionBatch.CorrelatedSubmission +import com.daml.ledger.participant.state.kvutils.wire._ import com.daml.ledger.participant.state.kvutils.{Envelope, KeyValueCommitting, Raw} -import com.daml.ledger.validator.ArgumentMatchers.{anyExecutionContext, iterableOf} +import com.daml.ledger.validator.ArgumentMatchers.{ + anyExecutionContext, + anyLoggingContext, + iterableOf, +} import com.daml.ledger.validator.TestHelper.{aParticipantId, anInvalidEnvelope, makePartySubmission} import com.daml.ledger.validator.batch.BatchedSubmissionValidatorSpec._ import com.daml.ledger.validator.reading.DamlLedgerStateReader @@ -23,6 +27,7 @@ import com.daml.ledger.validator.{CommitStrategy, ValidationFailed} import com.daml.lf.data.Ref import com.daml.lf.data.Time.Timestamp import com.daml.lf.engine.Engine +import com.daml.logging.LoggingContext import com.daml.metrics.Metrics import org.mockito.captor.ArgCaptor import org.mockito.{ArgumentMatchersSugar, MockitoSugar} @@ -131,7 +136,7 @@ class BatchedSubmissionValidatorSpec val mockCommit = mock[CommitStrategy[Unit]] val partySubmission = makePartySubmission("foo") // Expect two keys, i.e., to retrieve the party and submission dedup values. - when(mockLedgerStateReader.read(iterableOf(size = 2))(anyExecutionContext)) + when(mockLedgerStateReader.read(iterableOf(size = 2))(anyExecutionContext, anyLoggingContext)) .thenReturn(Future.successful(Seq(None, None))) val logEntryCaptor = ArgCaptor[DamlLogEntry] val outputStateCaptor = ArgCaptor[Map[DamlStateKey, DamlStateValue]] @@ -144,7 +149,7 @@ class BatchedSubmissionValidatorSpec any[Map[DamlStateKey, Option[DamlStateValue]]], outputStateCaptor.capture, any[Option[SubmissionAggregator.WriteSetBuilder]], - ) + )(any[LoggingContext]) ) .thenReturn(Future.unit) val validator = newBatchedSubmissionValidator[Unit]( @@ -178,7 +183,7 @@ class BatchedSubmissionValidatorSpec val (submissions, _, batchSubmissionBytes) = createBatchSubmissionOf(1000) val mockLedgerStateReader = mock[DamlLedgerStateReader] // Expect two keys, i.e., to retrieve the party and submission dedup values. - when(mockLedgerStateReader.read(iterableOf(size = 2))(anyExecutionContext)) + when(mockLedgerStateReader.read(iterableOf(size = 2))(anyExecutionContext, anyLoggingContext)) .thenReturn(Future.successful(Seq(None, None))) val logEntryCaptor = ArgCaptor[DamlLogEntry] val outputStateCaptor = ArgCaptor[Map[DamlStateKey, DamlStateValue]] @@ -192,7 +197,7 @@ class BatchedSubmissionValidatorSpec any[Map[DamlStateKey, Option[DamlStateValue]]], outputStateCaptor.capture, any[Option[SubmissionAggregator.WriteSetBuilder]], - ) + )(any[LoggingContext]) ) .thenReturn(Future.unit) val validator = @@ -210,7 +215,7 @@ class BatchedSubmissionValidatorSpec .map { _ => // We expected two state fetches and two commits. verify(mockLedgerStateReader, times(1000)) - .read(any[Seq[DamlStateKey]])(anyExecutionContext) + .read(any[Seq[DamlStateKey]])(anyExecutionContext, anyLoggingContext) verify(mockCommit, times(1000)).commit( any[Ref.ParticipantId], any[String], @@ -219,7 +224,7 @@ class BatchedSubmissionValidatorSpec any[DamlInputState], any[DamlOutputState], any[Option[SubmissionAggregator.WriteSetBuilder]], - ) + )(any[LoggingContext]) val actualEntries = logEntryCaptor.values.map(_.getPartyAllocationEntry) val expectedEntries = submissions.map(_.getPartyAllocationEntry) @@ -248,7 +253,7 @@ class BatchedSubmissionValidatorSpec .build() val mockLedgerStateReader = mock[DamlLedgerStateReader] // Expect two keys, i.e., to retrieve the party and submission dedup values. - when(mockLedgerStateReader.read(iterableOf(size = 2))(anyExecutionContext)) + when(mockLedgerStateReader.read(iterableOf(size = 2))(anyExecutionContext, anyLoggingContext)) .thenReturn(Future.successful(Seq(None, None))) val mockCommit = mock[CommitStrategy[Unit]] when( @@ -260,7 +265,7 @@ class BatchedSubmissionValidatorSpec any[Map[DamlStateKey, Option[DamlStateValue]]], any[Map[DamlStateKey, DamlStateValue]], any[Option[SubmissionAggregator.WriteSetBuilder]], - ) + )(any[LoggingContext]) ) .thenReturn(Future.unit) val validator = newBatchedSubmissionValidator[Unit]( @@ -286,7 +291,7 @@ class BatchedSubmissionValidatorSpec any[DamlInputState], any[DamlOutputState], any[Option[SubmissionAggregator.WriteSetBuilder]], - ) + )(any[LoggingContext]) succeed } } @@ -297,7 +302,7 @@ class BatchedSubmissionValidatorSpec val (submissions, batchSubmission, batchSubmissionBytes) = createBatchSubmissionOf(2) val mockLedgerStateReader = mock[DamlLedgerStateReader] // Expect two keys, i.e., to retrieve the party and submission dedup values. - when(mockLedgerStateReader.read(iterableOf(size = 2))(anyExecutionContext)) + when(mockLedgerStateReader.read(iterableOf(size = 2))(anyExecutionContext, anyLoggingContext)) .thenReturn(Future.successful(Seq(None, None))) val mockCommit = mock[CommitStrategy[Unit]] when( @@ -309,7 +314,7 @@ class BatchedSubmissionValidatorSpec any[Map[DamlStateKey, Option[DamlStateValue]]], any[Map[DamlStateKey, DamlStateValue]], any[Option[SubmissionAggregator.WriteSetBuilder]], - ) + )(any[LoggingContext]) ) .thenReturn(Future.unit) val validator = newBatchedSubmissionValidator[Unit]( diff --git a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/caching/CachingCommitStrategySpec.scala b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/caching/CachingCommitStrategySpec.scala index ca3011a8d9a4..b67459709fdf 100644 --- a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/caching/CachingCommitStrategySpec.scala +++ b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/caching/CachingCommitStrategySpec.scala @@ -16,6 +16,7 @@ import com.daml.ledger.validator.CommitStrategy import com.daml.ledger.validator.TestHelper._ import com.daml.ledger.validator.caching.CachingCommitStrategySpec._ import com.daml.lf.data.Ref +import com.daml.logging.LoggingContext import org.mockito.{ArgumentMatchersSugar, MockitoSugar} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AsyncWordSpec @@ -23,6 +24,8 @@ import org.scalatest.wordspec.AsyncWordSpec import scala.concurrent.{ExecutionContext, Future} class CachingCommitStrategySpec extends AsyncWordSpec with Matchers with MockitoSugar { + private implicit val loggingContext: LoggingContext = LoggingContext.ForTesting + "commit" should { "update cache with output state upon commit if policy allows" in { val cache = newCache() @@ -74,7 +77,7 @@ object CachingCommitStrategySpec { any[Map[DamlStateKey, Option[DamlStateValue]]], any[Map[DamlStateKey, DamlStateValue]], any[Option[SubmissionAggregator.WriteSetBuilder]], - ) + )(any[LoggingContext]) ) .thenReturn(Future.unit) new CachingCommitStrategy[Unit](cache, _ => shouldCache, mockCommitStrategy) diff --git a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/caching/CachingStateReaderSpec.scala b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/caching/CachingStateReaderSpec.scala index 667bc1bd9309..d32cb6a7d31d 100644 --- a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/caching/CachingStateReaderSpec.scala +++ b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/caching/CachingStateReaderSpec.scala @@ -5,9 +5,14 @@ package com.daml.ledger.validator.caching import com.daml.caching.Cache.Size import com.daml.caching.{Cache, Weight, WeightedCache} -import com.daml.ledger.validator.ArgumentMatchers.{anyExecutionContext, iterableOf} +import com.daml.ledger.validator.ArgumentMatchers.{ + anyExecutionContext, + anyLoggingContext, + iterableOf, +} import com.daml.ledger.validator.caching.CachingStateReaderSpec._ import com.daml.ledger.validator.reading.StateReader +import com.daml.logging.LoggingContext import org.mockito.{ArgumentMatchersSugar, MockitoSugar} import org.scalatest.Inside import org.scalatest.matchers.should.Matchers @@ -21,10 +26,12 @@ class CachingStateReaderSpec with Inside with MockitoSugar with ArgumentMatchersSugar { + private implicit val loggingContext: LoggingContext = LoggingContext.ForTesting + "read" should { "update cache upon read if policy allows" in { val mockReader = mock[TestStateReader] - when(mockReader.read(iterableOf(size = 1))(anyExecutionContext)) + when(mockReader.read(iterableOf(size = 1))(anyExecutionContext, anyLoggingContext)) .thenReturn(Future.successful(Seq(Some(TestValue.random())))) val (cache, instance) = newInstance(mockReader, shouldCacheOnRead = true) @@ -35,7 +42,7 @@ class CachingStateReaderSpec "do not update cache upon read if policy does not allow" in { val mockReader = mock[TestStateReader] - when(mockReader.read(iterableOf(size = 1))(anyExecutionContext)) + when(mockReader.read(iterableOf(size = 1))(anyExecutionContext, anyLoggingContext)) .thenReturn(Future.successful(Seq(Some(TestValue.random())))) val (cache, instance) = newInstance(mockReader, shouldCacheOnRead = false) @@ -46,7 +53,7 @@ class CachingStateReaderSpec "serve request from cache for seen key (if policy allows)" in { val mockReader = mock[TestStateReader] - when(mockReader.read(iterableOf(size = 1))(anyExecutionContext)) + when(mockReader.read(iterableOf(size = 1))(anyExecutionContext, anyLoggingContext)) .thenReturn(Future.successful(Seq(Some(TestValue(7))))) val (_, instance) = newInstance(mockReader, shouldCacheOnRead = true) @@ -54,7 +61,8 @@ class CachingStateReaderSpec originalReadState <- instance.read(Seq(TestKey(3))) readAgain <- instance.read(Seq(TestKey(3))) } yield { - verify(mockReader, times(1)).read(eqTo(Seq(TestKey(3))))(anyExecutionContext) + verify(mockReader, times(1)) + .read(eqTo(Seq(TestKey(3))))(anyExecutionContext, anyLoggingContext) readAgain shouldEqual originalReadState } } diff --git a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/preexecution/PreExecutingValidatingCommitterSpec.scala b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/preexecution/PreExecutingValidatingCommitterSpec.scala index 19709cc1932a..504f504353a8 100644 --- a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/preexecution/PreExecutingValidatingCommitterSpec.scala +++ b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/preexecution/PreExecutingValidatingCommitterSpec.scala @@ -41,18 +41,18 @@ class PreExecutingValidatingCommitterSpec fixture.conflictDetector.detectConflicts( any[PreExecutionOutput[TestReadSet, TestWriteSet]], any[StateReader[DamlStateKey, TestValue]], - )(any[ExecutionContext]) + )(any[ExecutionContext], any[LoggingContext]) ) .thenReturn(Future.unit) val submissionAggregator = mock[SubmissionAggregator] when(fixture.ledgerDataExporter.addSubmission(any[SubmissionInfo])) .thenReturn(submissionAggregator) when( - fixture.postExecutionWriter.write(any[TestWriteSet], any[LedgerStateWriteOperations[Long]])( - any[ExecutionContext] - ) - ) - .thenReturn(Future.successful(SubmissionResult.Acknowledged)) + fixture.postExecutionWriter.write( + any[TestWriteSet], + any[LedgerStateWriteOperations[Long]], + )(any[ExecutionContext], any[LoggingContext]) + ).thenReturn(Future.successful(SubmissionResult.Acknowledged)) fixture.committer .commit( aParticipantId, @@ -63,7 +63,10 @@ class PreExecutingValidatingCommitterSpec ) .map { result => verify(fixture.postExecutionWriter) - .write(any[TestWriteSet], any[LedgerStateWriteOperations[Long]])(any[ExecutionContext]) + .write(any[TestWriteSet], any[LedgerStateWriteOperations[Long]])( + any[ExecutionContext], + any[LoggingContext], + ) verify(submissionAggregator).finish() result shouldBe SubmissionResult.Acknowledged } @@ -76,9 +79,8 @@ class PreExecutingValidatingCommitterSpec fixture.conflictDetector.detectConflicts( any[PreExecutionOutput[TestReadSet, TestWriteSet]], any[StateReader[DamlStateKey, TestValue]], - )(any[ExecutionContext]) - ) - .thenReturn(Future.failed(new ConflictDetectedException())) + )(any[ExecutionContext], any[LoggingContext]) + ).thenReturn(Future.failed(new ConflictDetectedException())) val submissionAggregator = mock[SubmissionAggregator] when(fixture.ledgerDataExporter.addSubmission(any[SubmissionInfo])) .thenReturn(submissionAggregator) @@ -91,8 +93,10 @@ class PreExecutingValidatingCommitterSpec new FakeStateAccess(mock[LedgerStateOperations[Unit]]), ) .map { result => - verify(fixture.postExecutionWriter, never) - .write(any[TestWriteSet], any[LedgerStateWriteOperations[Long]])(any[ExecutionContext]) + verify(fixture.postExecutionWriter, never).write( + any[TestWriteSet], + any[LedgerStateWriteOperations[Long]], + )(any[ExecutionContext], any[LoggingContext]) verify(submissionAggregator, never).finish() result shouldBe SubmissionResult.Acknowledged } diff --git a/ledger/participant-state/kvutils/tools/BUILD.bazel b/ledger/participant-state/kvutils/tools/BUILD.bazel index be43369f1a7e..fa0fde648629 100644 --- a/ledger/participant-state/kvutils/tools/BUILD.bazel +++ b/ledger/participant-state/kvutils/tools/BUILD.bazel @@ -112,6 +112,7 @@ da_scala_binary( "//ledger/participant-state/kvutils", "//ledger/participant-state/kvutils:daml_kvutils_proto_java", "//ledger/participant-state/kvutils/tools", + "//libs-scala/contextualized-logging", "@maven//:com_google_protobuf_protobuf_java", ], ) @@ -140,6 +141,7 @@ da_scala_test( "//ledger/participant-state", "//ledger/participant-state/kvutils", "//ledger/participant-state/kvutils/tools", + "//libs-scala/contextualized-logging", "//libs-scala/logging-entries", "@maven//:com_google_protobuf_protobuf_java", "@maven//:io_dropwizard_metrics_metrics_core", diff --git a/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/LogAppendingCommitStrategySupport.scala b/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/LogAppendingCommitStrategySupport.scala index 51a394efbfd9..aa68d27465a0 100644 --- a/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/LogAppendingCommitStrategySupport.scala +++ b/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/LogAppendingCommitStrategySupport.scala @@ -19,6 +19,7 @@ import com.daml.ledger.validator.batch.{ ConflictDetection, } import com.daml.lf.engine.Engine +import com.daml.logging.LoggingContext import com.daml.metrics.Metrics import scala.concurrent.{ExecutionContext, Future} @@ -46,7 +47,7 @@ final class LogAppendingCommitStrategySupport( override def commit( submissionInfo: SubmissionInfo - )(implicit materializer: Materializer): Future[WriteSet] = { + )(implicit materializer: Materializer, loggingContext: LoggingContext): Future[WriteSet] = { val access = new WriteRecordingLedgerStateAccess(new InMemoryLedgerStateAccess(state, metrics)) access.inTransaction { operations => val (ledgerStateReader, commitStrategy) = diff --git a/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/RawPreExecutingCommitStrategySupport.scala b/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/RawPreExecutingCommitStrategySupport.scala index e7d0d6b0bf1b..c9cf09b0e8c6 100644 --- a/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/RawPreExecutingCommitStrategySupport.scala +++ b/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/RawPreExecutingCommitStrategySupport.scala @@ -9,7 +9,12 @@ import java.util.concurrent.atomic.AtomicReference import akka.stream.Materializer import com.daml.ledger.on.memory.{InMemoryLedgerStateAccess, InMemoryState, Index} import com.daml.ledger.participant.state.kvutils.DamlKvutils.DamlStateValue -import com.daml.ledger.participant.state.kvutils.{KeyValueCommitting, export} +import com.daml.ledger.participant.state.kvutils.KeyValueCommitting +import com.daml.ledger.participant.state.kvutils.export.{ + NoOpLedgerDataExporter, + SubmissionInfo, + WriteSet, +} import com.daml.ledger.validator.preexecution.{ EqualityBasedPostExecutionConflictDetector, PreExecutingSubmissionValidator, @@ -21,6 +26,7 @@ import com.daml.ledger.validator.preexecution.{ } import com.daml.ledger.validator.{SerializingStateReader, StateKeySerializationStrategy} import com.daml.lf.engine.Engine +import com.daml.logging.LoggingContext import com.daml.metrics.Metrics import scala.concurrent.{ExecutionContext, Future} @@ -60,12 +66,12 @@ final class RawPreExecutingCommitStrategySupport( postExecutionConflictDetector = new EqualityBasedPostExecutionConflictDetector, postExecutionWriteSetSelector = postExecutionWriteSetSelector, postExecutionWriter = new RawPostExecutionWriter, - ledgerDataExporter = export.NoOpLedgerDataExporter, + ledgerDataExporter = NoOpLedgerDataExporter, ) override def commit( - submissionInfo: export.SubmissionInfo - )(implicit materializer: Materializer): Future[export.WriteSet] = { + submissionInfo: SubmissionInfo + )(implicit materializer: Materializer, loggingContext: LoggingContext): Future[WriteSet] = { val access = new WriteRecordingLedgerStateAccess(ledgerStateAccess) currentSubmissionRecordTime.set(submissionInfo.recordTimeInstant) committer diff --git a/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/WriteRecordingLedgerStateAccess.scala b/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/WriteRecordingLedgerStateAccess.scala index 38dd677672ff..752da9382b02 100644 --- a/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/WriteRecordingLedgerStateAccess.scala +++ b/ledger/participant-state/kvutils/tools/integrity-check/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/WriteRecordingLedgerStateAccess.scala @@ -6,6 +6,7 @@ package com.daml.ledger.participant.state.kvutils.tools.integritycheck import com.daml.ledger.participant.state.kvutils.Raw import com.daml.ledger.participant.state.kvutils.export.{WriteItem, WriteSet} import com.daml.ledger.validator.{LedgerStateAccess, LedgerStateOperations} +import com.daml.logging.LoggingContext import scala.collection.mutable import scala.concurrent.{ExecutionContext, Future} @@ -16,7 +17,7 @@ class WriteRecordingLedgerStateAccess[LogResult](delegate: LedgerStateAccess[Log override def inTransaction[T]( body: LedgerStateOperations[LogResult] => Future[T] - )(implicit executionContext: ExecutionContext): Future[T] = + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[T] = delegate.inTransaction { operations => body(new WriteRecordingLedgerStateAccess.Operations(recordedWriteSet, operations)) } @@ -32,25 +33,31 @@ object WriteRecordingLedgerStateAccess { ) extends LedgerStateOperations[LogResult] { override def readState( key: Raw.StateKey - )(implicit executionContext: ExecutionContext): Future[Option[Raw.Envelope]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Option[Raw.Envelope]] = delegate.readState(key) override def readState( keys: Iterable[Raw.StateKey] - )(implicit executionContext: ExecutionContext): Future[Seq[Option[Raw.Envelope]]] = + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Seq[Option[Raw.Envelope]]] = delegate.readState(keys) override def writeState( key: Raw.StateKey, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[Unit] = { + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = { this.synchronized(recordedWriteSet.append((key, value))) delegate.writeState(key, value) } override def writeState( keyValuePairs: Iterable[Raw.StateEntry] - )(implicit executionContext: ExecutionContext): Future[Unit] = { + )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext): Future[Unit] = { this.synchronized(recordedWriteSet.appendAll(keyValuePairs)) delegate.writeState(keyValuePairs) } @@ -58,7 +65,10 @@ object WriteRecordingLedgerStateAccess { override def appendToLog( key: Raw.LogEntryId, value: Raw.Envelope, - )(implicit executionContext: ExecutionContext): Future[LogResult] = { + )(implicit + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[LogResult] = { this.synchronized(recordedWriteSet.append((key, value))) delegate.appendToLog(key, value) } diff --git a/ledger/participant-state/kvutils/tools/integrity-check/src/test/scala/ledger/participant/state/kvutils/tools/integritycheck/RawPreExecutingCommitStrategySupportSpec.scala b/ledger/participant-state/kvutils/tools/integrity-check/src/test/scala/ledger/participant/state/kvutils/tools/integritycheck/RawPreExecutingCommitStrategySupportSpec.scala index 57abd5cb63cb..e2e9fe3231a1 100644 --- a/ledger/participant-state/kvutils/tools/integrity-check/src/test/scala/ledger/participant/state/kvutils/tools/integritycheck/RawPreExecutingCommitStrategySupportSpec.scala +++ b/ledger/participant-state/kvutils/tools/integrity-check/src/test/scala/ledger/participant/state/kvutils/tools/integritycheck/RawPreExecutingCommitStrategySupportSpec.scala @@ -20,6 +20,7 @@ import com.daml.ledger.participant.state.kvutils.tools.integritycheck.RawPreExec import com.daml.ledger.participant.state.kvutils.wire.DamlSubmission import com.daml.ledger.participant.state.kvutils.{Envelope, Raw} import com.daml.lf.data.Ref +import com.daml.logging.LoggingContext import com.daml.metrics.Metrics import com.google.protobuf.{Empty, Timestamp} import org.scalatest.matchers.should.Matchers @@ -29,6 +30,8 @@ class RawPreExecutingCommitStrategySupportSpec extends AsyncWordSpec with Matchers with AkkaBeforeAndAfterAll { + private implicit val loggingContext: LoggingContext = LoggingContext.ForTesting + "support" should { "commit, and provide the write set" in { val metrics = new Metrics(new MetricRegistry) diff --git a/ledger/participant-state/kvutils/tools/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/CommitStrategySupport.scala b/ledger/participant-state/kvutils/tools/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/CommitStrategySupport.scala index 3c6aea43c57c..a01d85356053 100644 --- a/ledger/participant-state/kvutils/tools/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/CommitStrategySupport.scala +++ b/ledger/participant-state/kvutils/tools/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/CommitStrategySupport.scala @@ -6,6 +6,7 @@ package com.daml.ledger.participant.state.kvutils.tools.integritycheck import akka.stream.Materializer import com.daml.ledger.participant.state.kvutils.export.{SubmissionInfo, WriteSet} import com.daml.ledger.validator.StateKeySerializationStrategy +import com.daml.logging.LoggingContext import scala.concurrent.Future @@ -14,7 +15,7 @@ trait CommitStrategySupport[LogResult] { def commit( submissionInfo: SubmissionInfo - )(implicit materializer: Materializer): Future[WriteSet] + )(implicit materializer: Materializer, loggingContext: LoggingContext): Future[WriteSet] def newReadServiceFactory(): ReplayingReadServiceFactory diff --git a/ledger/participant-state/kvutils/tools/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/IntegrityChecker.scala b/ledger/participant-state/kvutils/tools/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/IntegrityChecker.scala index 965f4fcb0a14..96764949952e 100644 --- a/ledger/participant-state/kvutils/tools/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/IntegrityChecker.scala +++ b/ledger/participant-state/kvutils/tools/src/main/scala/ledger/participant/state/kvutils/tools/integritycheck/IntegrityChecker.scala @@ -48,7 +48,11 @@ class IntegrityChecker[LogResult]( def run( importer: LedgerDataImporter, config: Config, - )(implicit executionContext: ExecutionContext, materializer: Materializer): Future[Unit] = { + )(implicit + executionContext: ExecutionContext, + materializer: Materializer, + loggingContext: LoggingContext, + ): Future[Unit] = { if (config.indexOnly) println("Running indexing only".white) @@ -93,6 +97,7 @@ class IntegrityChecker[LogResult]( )(implicit executionContext: ExecutionContext, materializer: Materializer, + loggingContext: LoggingContext, ): Future[Unit] = for { _ <- processSubmissions( @@ -186,7 +191,11 @@ class IntegrityChecker[LogResult]( expectedReadServiceFactory: ReplayingReadServiceFactory, actualReadServiceFactory: ReplayingReadServiceFactory, config: Config, - )(implicit materializer: Materializer, executionContext: ExecutionContext): Future[Unit] = { + )(implicit + materializer: Materializer, + executionContext: ExecutionContext, + loggingContext: LoggingContext, + ): Future[Unit] = { println("Processing the ledger export.".white) Source(importer.read()) @@ -309,19 +318,21 @@ object IntegrityChecker { ExecutionContext.fromExecutorService(Executors.newCachedThreadPool()) implicit val materializer: Materializer = Materializer(actorSystem) - val importer = ProtobufBasedLedgerDataImporter(config.exportFilePath) - new IntegrityChecker(commitStrategySupportFactory(_, executionContext)) - .run(importer, config) - .onComplete { - case Success(_) => - sys.exit(0) - case Failure(exception: CheckFailedException) => - println(exception.getMessage.red) - sys.exit(1) - case Failure(exception) => - exception.printStackTrace() - sys.exit(1) - }(DirectExecutionContext) + newLoggingContext { implicit loggingContext => + val importer = ProtobufBasedLedgerDataImporter(config.exportFilePath) + new IntegrityChecker(commitStrategySupportFactory(_, executionContext)) + .run(importer, config) + .onComplete { + case Success(_) => + sys.exit(0) + case Failure(exception: CheckFailedException) => + println(exception.getMessage.red) + sys.exit(1) + case Failure(exception) => + exception.printStackTrace() + sys.exit(1) + }(DirectExecutionContext) + } } private[integritycheck] def createIndexerConfig(config: Config): IndexerConfig = From 12e0c72d5c7f138f93cd0e50eea87c32f426be86 Mon Sep 17 00:00:00 2001 From: Gary Verhaegen Date: Tue, 28 Sep 2021 16:28:19 +0200 Subject: [PATCH 7/9] fix blackduck logic (#11049) We believe the Blakduck logic is currently faulty. We have had a violation on an NPM dependency, and Blackduck keeps reporting it despite our having removed the dependency. We believe that what is happening is that, in the first step of checking, we udpate the Haskell dependencies, _and then check the validity of the whole project_, which includes the NPM deps. Because that fails, we never get to the step where we actually update the NPM deps, and Blackduck is stuck forever. The solution is to not fail on violations for the Haskell update steps. Haskell deps are still checked in the second step, because, again, it is checking the whole project. CHANGELOG_BEGIN CHANGELOG_END --- ci/cron/daily-compat.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/cron/daily-compat.yml b/ci/cron/daily-compat.yml index fab4eaa2588e..7d9bdfcf8016 100644 --- a/ci/cron/daily-compat.yml +++ b/ci/cron/daily-compat.yml @@ -382,7 +382,6 @@ jobs: --detect.tools=BAZEL \ --detect.bazel.target=//... \ --detect.bazel.dependency.type=${BAZEL_DEPENDENCY_TYPE} \ - --detect.policy.check.fail.on.severities=MAJOR,CRITICAL,BLOCKER \ --detect.notices.report=true \ --detect.code.location.name=digital-asset_daml_${BAZEL_DEPENDENCY_TYPE} \ --detect.timeout=1500 From a885f52c4df6fb3a6b19de4ef57fed9eade189f9 Mon Sep 17 00:00:00 2001 From: pbatko-da Date: Tue, 28 Sep 2021 16:30:55 +0200 Subject: [PATCH 8/9] [DPP-417] [DPP-595] Add error code version switching mechanism (#11035) Adding: - a mechanism to switch between error code versions: `ErrorCodesVersionSwitcher`, - the corresponding CLI flag `--use-error-codes-v2` for sandbox-classic, sandbox and ledger-on-X. --- As of this PR: - `ErrorCodesVersionSwitcher` is never used in production code (apart from a no-op usage in `ApiTransactionService` to prove the point), - `--use-error-codes-v2` flag is set to hidden (until new error have been adopted). These will be used in the subsequent stories from the same epic: https://digitalasset.atlassian.net/browse/DPP-417 --- .../platform/apiserver/ApiServerConfig.scala | 8 ++-- .../platform/apiserver/ApiServices.scala | 9 +++-- .../apiserver/ErrorCodesVersionSwitcher.scala | 19 +++++++++ .../apiserver/StandaloneApiServer.scala | 1 + .../apiserver/error/TransactionError.scala | 6 +++ .../transaction/ApiTransactionService.scala | 26 +++++++++--- .../ErrorCodesVersionSwitcherSpec.scala | 40 +++++++++++++++++++ .../state/kvutils/app/Config.scala | 8 ++++ .../state/kvutils/app/LedgerFactory.scala | 1 + .../state/kvutils/app/ConfigSpec.scala | 28 ++++++++++++- .../platform/sandbox/SandboxServer.scala | 1 + .../platform/sandbox/cli/CommonCliBase.scala | 6 +++ .../sandbox/config/SandboxConfig.scala | 2 + .../sandbox/cli/CommonCliSpecBase.scala | 12 ++++++ .../scala/platform/sandboxnext/Runner.scala | 1 + 15 files changed, 153 insertions(+), 15 deletions(-) create mode 100644 ledger/participant-integration-api/src/main/scala/platform/apiserver/ErrorCodesVersionSwitcher.scala create mode 100644 ledger/participant-integration-api/src/test/suite/scala/platform/apiserver/ErrorCodesVersionSwitcherSpec.scala diff --git a/ledger/participant-integration-api/src/main/scala/platform/apiserver/ApiServerConfig.scala b/ledger/participant-integration-api/src/main/scala/platform/apiserver/ApiServerConfig.scala index 463b97ebacbc..51f5db04be6b 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/apiserver/ApiServerConfig.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/apiserver/ApiServerConfig.scala @@ -3,16 +3,15 @@ package com.daml.platform.apiserver -import java.io.File -import java.nio.file.Path -import java.time.Duration - import com.daml.ledger.api.tls.TlsConfiguration import com.daml.lf.data.Ref import com.daml.platform.apiserver.SeedService.Seeding import com.daml.platform.configuration.{IndexConfiguration, InitialLedgerConfiguration} import com.daml.ports.Port +import java.io.File +import java.nio.file.Path +import java.time.Duration import scala.concurrent.duration.FiniteDuration case class ApiServerConfig( @@ -39,4 +38,5 @@ case class ApiServerConfig( enableMutableContractStateCache: Boolean, maxTransactionsInMemoryFanOutBufferSize: Long, enableInMemoryFanOutForLedgerApi: Boolean, + enableErrorCodesV2: Boolean, ) diff --git a/ledger/participant-integration-api/src/main/scala/platform/apiserver/ApiServices.scala b/ledger/participant-integration-api/src/main/scala/platform/apiserver/ApiServices.scala index 3c01867b8275..9190cf9c1a29 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/apiserver/ApiServices.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/apiserver/ApiServices.scala @@ -3,8 +3,6 @@ package com.daml.platform.apiserver -import java.time.Duration - import akka.stream.Materializer import com.daml.api.util.TimeProvider import com.daml.grpc.adapter.ExecutionSequencerFactory @@ -54,6 +52,7 @@ import io.grpc.BindableService import io.grpc.protobuf.services.ProtoReflectionService import scalaz.syntax.tag._ +import java.time.Duration import scala.collection.immutable import scala.concurrent.duration.{Duration => ScalaDuration} import scala.concurrent.{ExecutionContext, Future} @@ -94,6 +93,7 @@ private[daml] object ApiServices { healthChecks: HealthChecks, seedService: SeedService, managementServiceTimeout: Duration, + enableErrorCodesV2: Boolean, )(implicit materializer: Materializer, esf: ExecutionSequencerFactory, @@ -118,6 +118,9 @@ private[daml] object ApiServices { servicesExecutionContext = servicesExecutionContext, ) + private val errorsVersionsSwitcher = + new ErrorCodesVersionSwitcher(enableErrorCodesV2 = enableErrorCodesV2) + override def acquire()(implicit context: ResourceContext): Resource[ApiServices] = { logger.info(engine.info.toString) for { @@ -144,7 +147,7 @@ private[daml] object ApiServices { ledgerConfigurationSubscription: LedgerConfigurationSubscription, )(implicit executionContext: ExecutionContext): List[BindableService] = { val apiTransactionService = - ApiTransactionService.create(ledgerId, transactionsService, metrics) + ApiTransactionService.create(ledgerId, transactionsService, metrics, errorsVersionsSwitcher) val apiLedgerIdentityService = ApiLedgerIdentityService.create(() => identityService.getLedgerId()) diff --git a/ledger/participant-integration-api/src/main/scala/platform/apiserver/ErrorCodesVersionSwitcher.scala b/ledger/participant-integration-api/src/main/scala/platform/apiserver/ErrorCodesVersionSwitcher.scala new file mode 100644 index 000000000000..14d84647834e --- /dev/null +++ b/ledger/participant-integration-api/src/main/scala/platform/apiserver/ErrorCodesVersionSwitcher.scala @@ -0,0 +1,19 @@ +// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.daml.platform.apiserver + +import io.grpc.StatusRuntimeException + +final class ErrorCodesVersionSwitcher(enableErrorCodesV2: Boolean) { + def choose( + v1: => StatusRuntimeException, + v2: => StatusRuntimeException, + ): StatusRuntimeException = { + if (enableErrorCodesV2) { + v2 + } else { + v1 + } + } +} diff --git a/ledger/participant-integration-api/src/main/scala/platform/apiserver/StandaloneApiServer.scala b/ledger/participant-integration-api/src/main/scala/platform/apiserver/StandaloneApiServer.scala index 2918f3a8a51a..735ebb42a8f7 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/apiserver/StandaloneApiServer.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/apiserver/StandaloneApiServer.scala @@ -117,6 +117,7 @@ final class StandaloneApiServer( healthChecks = healthChecksWithIndexService, seedService = SeedService(config.seeding), managementServiceTimeout = config.managementServiceTimeout, + enableErrorCodesV2 = config.enableErrorCodesV2, )(materializer, executionSequencerFactory, loggingContext) .map(_.withServices(otherServices)) apiServer <- new LedgerApiServer( diff --git a/ledger/participant-integration-api/src/main/scala/platform/apiserver/error/TransactionError.scala b/ledger/participant-integration-api/src/main/scala/platform/apiserver/error/TransactionError.scala index 2b897d414f19..40e325cd9ede 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/apiserver/error/TransactionError.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/apiserver/error/TransactionError.scala @@ -11,6 +11,7 @@ import com.daml.ledger.participant.state.v2.Update.CommandRejected.{ } import com.daml.logging.{ContextualizedLogger, LoggingContext} import com.google.rpc.status.{Status => RpcStatus} +import io.grpc.StatusRuntimeException trait TransactionError extends BaseError { def createRejection( @@ -106,6 +107,11 @@ abstract class LoggingTransactionErrorImpl( correlationId: CorrelationId, ) extends TransactionErrorImpl(cause, throwableO, definiteAnswer)(code) { + def asGrpcError: StatusRuntimeException = asGrpcErrorFromContext( + correlationId = correlationId.id, + logger = logger, + )(loggingContext) + def log(): Unit = logWithContext(logger, correlationId.id)(loggingContext) // Automatically log the error on generation diff --git a/ledger/participant-integration-api/src/main/scala/platform/apiserver/services/transaction/ApiTransactionService.scala b/ledger/participant-integration-api/src/main/scala/platform/apiserver/services/transaction/ApiTransactionService.scala index d5663d902511..81e59caf5dd9 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/apiserver/services/transaction/ApiTransactionService.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/apiserver/services/transaction/ApiTransactionService.scala @@ -29,6 +29,8 @@ import com.daml.logging.LoggingContext.withEnrichedLoggingContext import com.daml.logging.entries.LoggingEntries import com.daml.logging.{ContextualizedLogger, LoggingContext} import com.daml.metrics.Metrics +import com.daml.platform.apiserver.ErrorCodesVersionSwitcher +import com.daml.platform.apiserver.error.{CorrelationId, LedgerApiErrors} import com.daml.platform.apiserver.services.transaction.ApiTransactionService._ import com.daml.platform.apiserver.services.{StreamMetrics, logging} import com.daml.platform.server.api.services.domain.TransactionService @@ -44,6 +46,7 @@ private[apiserver] object ApiTransactionService { ledgerId: LedgerId, transactionsService: IndexTransactionsService, metrics: Metrics, + errorsVersionsSwitcher: ErrorCodesVersionSwitcher, )(implicit ec: ExecutionContext, mat: Materializer, @@ -51,7 +54,7 @@ private[apiserver] object ApiTransactionService { loggingContext: LoggingContext, ): GrpcTransactionService with BindableService = new GrpcTransactionService( - new ApiTransactionService(transactionsService, metrics), + new ApiTransactionService(transactionsService, metrics, errorsVersionsSwitcher), ledgerId, PartyNameChecker.AllowAllParties, ) @@ -68,6 +71,7 @@ private[apiserver] object ApiTransactionService { private[apiserver] final class ApiTransactionService private ( transactionsService: IndexTransactionsService, metrics: Metrics, + errorsVersionsSwitcher: ErrorCodesVersionSwitcher, )(implicit executionContext: ExecutionContext, loggingContext: LoggingContext) extends TransactionService with ErrorFactories { @@ -137,13 +141,23 @@ private[apiserver] final class ApiTransactionService private ( .map { case LfEventId(transactionId, _) => lookUpTreeByTransactionId(TransactionId(transactionId), request.requestingParties) } - .getOrElse( + .getOrElse { + val msg = s"invalid eventId: ${request.eventId}" Future.failed( - Status.NOT_FOUND - .withDescription(s"invalid eventId: ${request.eventId}") - .asRuntimeException() + errorsVersionsSwitcher.choose( + v1 = Status.NOT_FOUND + .withDescription(msg) + .asRuntimeException(), + v2 = LedgerApiErrors.CommandValidation.InvalidArgument + .Reject(msg)( + correlationId = CorrelationId.none, + loggingContext = implicitly[LoggingContext], + logger = logger, + ) + .asGrpcError, + ) ) - ) + } .andThen(logger.logErrorsOnCall[GetTransactionResponse]) } diff --git a/ledger/participant-integration-api/src/test/suite/scala/platform/apiserver/ErrorCodesVersionSwitcherSpec.scala b/ledger/participant-integration-api/src/test/suite/scala/platform/apiserver/ErrorCodesVersionSwitcherSpec.scala new file mode 100644 index 000000000000..bdea72c33b38 --- /dev/null +++ b/ledger/participant-integration-api/src/test/suite/scala/platform/apiserver/ErrorCodesVersionSwitcherSpec.scala @@ -0,0 +1,40 @@ +// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.daml.platform.apiserver + +import io.grpc.{Status, StatusRuntimeException} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class ErrorCodesVersionSwitcherSpec extends AnyFlatSpec with Matchers { + + behavior of classOf[ErrorCodesVersionSwitcher].getSimpleName + + it should "use v2 error codes" in { + // given + val tested = new ErrorCodesVersionSwitcher(enableErrorCodesV2 = true) + val expected = new StatusRuntimeException(Status.INTERNAL) + + // when + val actual = + tested.choose(v1 = fail("This argument should be evaluated lazily!"), v2 = expected) + + // then + actual shouldBe expected + } + + it should "use v1 error codes" in { + // given + val tested = new ErrorCodesVersionSwitcher(enableErrorCodesV2 = false) + val expected = new StatusRuntimeException(Status.INTERNAL) + + // when + val actual = + tested.choose(v1 = expected, v2 = fail("This argument should be evaluated lazily!")) + + // then + actual shouldBe expected + } + +} diff --git a/ledger/participant-state/kvutils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/Config.scala b/ledger/participant-state/kvutils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/Config.scala index 31ddc025ac87..1ac6deb4ae25 100644 --- a/ledger/participant-state/kvutils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/Config.scala +++ b/ledger/participant-state/kvutils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/Config.scala @@ -54,6 +54,7 @@ final case class Config[Extra]( enableInMemoryFanOutForLedgerApi: Boolean, enableHa: Boolean, // TODO ha: remove after stable extra: Extra, + enableErrorCodesV2: Boolean, ) { def withTlsConfig(modify: TlsConfiguration => TlsConfiguration): Config[Extra] = copy(tlsConfig = Some(modify(tlsConfig.getOrElse(TlsConfiguration.Empty)))) @@ -90,6 +91,7 @@ object Config { enableHa = false, maxDeduplicationDuration = None, extra = extra, + enableErrorCodesV2 = false, ) def ownerWithoutExtras(name: String, args: collection.Seq[String]): ResourceOwner[Config[Unit]] = @@ -657,6 +659,12 @@ object Config { s"Use the experimental High Availability feature with the indexer. Should not be used in production." ) .action((_, config) => config.copy(enableHa = true)) + + opt[Unit]("use-error-codes-v2") + .optional() + .hidden() + .text("Enable new self-service errors.") + .action((_, config) => config.copy(enableErrorCodesV2 = true)) } extraOptions(parser) parser diff --git a/ledger/participant-state/kvutils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/LedgerFactory.scala b/ledger/participant-state/kvutils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/LedgerFactory.scala index 7957b8f2e879..87c20c016b9a 100644 --- a/ledger/participant-state/kvutils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/LedgerFactory.scala +++ b/ledger/participant-state/kvutils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/LedgerFactory.scala @@ -88,6 +88,7 @@ trait ConfigProvider[ExtraConfig] { maxTransactionsInMemoryFanOutBufferSize = participantConfig.maxTransactionsInMemoryFanOutBufferSize, enableInMemoryFanOutForLedgerApi = config.enableInMemoryFanOutForLedgerApi, + enableErrorCodesV2 = config.enableErrorCodesV2, ) def partyConfig(@unused config: Config[ExtraConfig]): PartyConfiguration = diff --git a/ledger/participant-state/kvutils/app/src/test/scala/com/daml/ledger/participant/state/kvutils/app/ConfigSpec.scala b/ledger/participant-state/kvutils/app/src/test/scala/com/daml/ledger/participant/state/kvutils/app/ConfigSpec.scala index 8cc5e8df1009..1814a2571b83 100644 --- a/ledger/participant-state/kvutils/app/src/test/scala/com/daml/ledger/participant/state/kvutils/app/ConfigSpec.scala +++ b/ledger/participant-state/kvutils/app/src/test/scala/com/daml/ledger/participant/state/kvutils/app/ConfigSpec.scala @@ -3,8 +3,6 @@ package com.daml.ledger.participant.state.kvutils.app -import java.io.File -import java.time.Duration import com.daml.ledger.api.tls.{SecretsUrl, TlsConfiguration, TlsVersion} import com.daml.lf.data.Ref import io.netty.handler.ssl.ClientAuth @@ -14,6 +12,9 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks import scopt.OptionParser +import java.io.File +import java.time.Duration + final class ConfigSpec extends AnyFlatSpec with Matchers @@ -58,6 +59,29 @@ final class ConfigSpec behavior of "Runner" + it should "parse error codes v2 flag" in { + val actual = configParser( + Seq( + dumpIndexMetadataCommand, + "some-jdbc-url", + "--use-error-codes-v2", + ) + ) + + actual.value.enableErrorCodesV2 shouldBe true + } + + it should "disable error codes v2 flag by default" in { + val actual = configParser( + Seq( + dumpIndexMetadataCommand, + "some-jdbc-url", + ) + ) + + actual.value.enableErrorCodesV2 shouldBe false + } + it should "succeed when server's private key is encrypted and secret-url is provided" in { val actual = configParser( Seq( diff --git a/ledger/sandbox-classic/src/main/scala/platform/sandbox/SandboxServer.scala b/ledger/sandbox-classic/src/main/scala/platform/sandbox/SandboxServer.scala index 139a8947c2dc..64aaea0ab43d 100644 --- a/ledger/sandbox-classic/src/main/scala/platform/sandbox/SandboxServer.scala +++ b/ledger/sandbox-classic/src/main/scala/platform/sandbox/SandboxServer.scala @@ -386,6 +386,7 @@ final class SandboxServer( healthChecks = healthChecks, seedService = seedingService, managementServiceTimeout = config.managementServiceTimeout, + enableErrorCodesV2 = config.enableErrorCodesV2, )(materializer, executionSequencerFactory, loggingContext) .map(_.withServices(List(resetService))) apiServer <- new LedgerApiServer( diff --git a/ledger/sandbox-common/src/main/scala/platform/sandbox/cli/CommonCliBase.scala b/ledger/sandbox-common/src/main/scala/platform/sandbox/cli/CommonCliBase.scala index 5e0af325e24e..84c2e4067ac6 100644 --- a/ledger/sandbox-common/src/main/scala/platform/sandbox/cli/CommonCliBase.scala +++ b/ledger/sandbox-common/src/main/scala/platform/sandbox/cli/CommonCliBase.scala @@ -354,6 +354,12 @@ class CommonCliBase(name: LedgerName) { "Maximum command deduplication duration." ) + opt[Unit]("use-error-codes-v2") + .optional() + .hidden() + .text("Enable new self-service errors.") + .action((_, config) => config.copy(enableErrorCodesV2 = true)) + checkConfig(c => { if (c.enableCompression && !c.enableAppendOnlySchema) failure( diff --git a/ledger/sandbox-common/src/main/scala/platform/sandbox/config/SandboxConfig.scala b/ledger/sandbox-common/src/main/scala/platform/sandbox/config/SandboxConfig.scala index 7d25ad8056a7..69ea82ea7fdd 100644 --- a/ledger/sandbox-common/src/main/scala/platform/sandbox/config/SandboxConfig.scala +++ b/ledger/sandbox-common/src/main/scala/platform/sandbox/config/SandboxConfig.scala @@ -66,6 +66,7 @@ final case class SandboxConfig( sqlStartMode: Option[PostgresStartupMode], enableAppendOnlySchema: Boolean, enableCompression: Boolean, + enableErrorCodesV2: Boolean, ) { def withTlsConfig(modify: TlsConfiguration => TlsConfiguration): SandboxConfig = @@ -146,6 +147,7 @@ object SandboxConfig { sqlStartMode = Some(DefaultSqlStartupMode), enableAppendOnlySchema = false, enableCompression = false, + enableErrorCodesV2 = false, ) sealed abstract class EngineMode extends Product with Serializable diff --git a/ledger/sandbox-common/src/test/lib/scala/platform/sandbox/cli/CommonCliSpecBase.scala b/ledger/sandbox-common/src/test/lib/scala/platform/sandbox/cli/CommonCliSpecBase.scala index ed1631c5259d..fc042d4e8204 100644 --- a/ledger/sandbox-common/src/test/lib/scala/platform/sandbox/cli/CommonCliSpecBase.scala +++ b/ledger/sandbox-common/src/test/lib/scala/platform/sandbox/cli/CommonCliSpecBase.scala @@ -356,6 +356,18 @@ abstract class CommonCliSpecBase( _.copy(metricsReportingInterval = 90.seconds), ) } + + "parse error codes v2 flag" in { + checkOption( + Array("--use-error-codes-v2"), + _.copy(enableErrorCodesV2 = true), + ) + checkOption( + Array(), + _.copy(enableErrorCodesV2 = false), + ) + } + } protected def checkOption( diff --git a/ledger/sandbox/src/main/scala/platform/sandboxnext/Runner.scala b/ledger/sandbox/src/main/scala/platform/sandboxnext/Runner.scala index 0ccc674cb719..f7fe284f2075 100644 --- a/ledger/sandbox/src/main/scala/platform/sandboxnext/Runner.scala +++ b/ledger/sandbox/src/main/scala/platform/sandboxnext/Runner.scala @@ -257,6 +257,7 @@ class Runner(config: SandboxConfig) extends ResourceOwner[Port] { enableMutableContractStateCache = false, maxTransactionsInMemoryFanOutBufferSize = 0L, enableInMemoryFanOutForLedgerApi = false, + enableErrorCodesV2 = config.enableErrorCodesV2, ), engine = engine, commandConfig = config.commandConfig, From 9641fd5f83f5954add9fc147c9bd43e5a1abd885 Mon Sep 17 00:00:00 2001 From: Gary Verhaegen Date: Tue, 28 Sep 2021 17:00:01 +0200 Subject: [PATCH 9/9] auth middleware: no print secret (#11050) At the moment, the OAuth2 Middleware prints its config at startup. Unfortunately that config includes a secret, which presumably we'd prefer not to show. CHANGELOG_BEGIN - The OAuth2 Middleware now obfuscates its Client Secret when logging its config. CHANGELOG_END --- .../daml/auth/middleware/oauth2/Config.scala | 8 +++++-- .../middleware/oauth2/RequestTemplates.scala | 24 ++++++++----------- .../auth/middleware/oauth2/TestFixture.scala | 2 +- .../oauth2/TestRequestTemplates.scala | 6 ++--- .../trigger/TriggerServiceFixture.scala | 8 +++++-- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/triggers/service/auth/src/main/scala/com/daml/auth/middleware/oauth2/Config.scala b/triggers/service/auth/src/main/scala/com/daml/auth/middleware/oauth2/Config.scala index 1c1200b4fcda..9e58303901c7 100644 --- a/triggers/service/auth/src/main/scala/com/daml/auth/middleware/oauth2/Config.scala +++ b/triggers/service/auth/src/main/scala/com/daml/auth/middleware/oauth2/Config.scala @@ -32,11 +32,15 @@ case class Config( oauthRefreshTemplate: Option[Path], // OAuth2 client properties clientId: String, - clientSecret: String, + clientSecret: SecretString, // Token verification tokenVerifier: JwtVerifierBase, ) +case class SecretString(value: String) { + override def toString: String = "###" +} + object Config { val DefaultHttpPort: Int = 3000 val DefaultCookieSecure: Boolean = true @@ -134,7 +138,7 @@ object Config { opt[String]("secret") .hidden() - .action((x, c) => c.copy(clientSecret = x)) + .action((x, c) => c.copy(clientSecret = SecretString(x))) .withFallback(() => sys.env.getOrElse("DAML_CLIENT_SECRET", "")) .validate(x => if (x.isEmpty) failure("Environment variable DAML_CLIENT_SECRET must not be empty") diff --git a/triggers/service/auth/src/main/scala/com/daml/auth/middleware/oauth2/RequestTemplates.scala b/triggers/service/auth/src/main/scala/com/daml/auth/middleware/oauth2/RequestTemplates.scala index e8e3cf0c8ab7..165929b9ddeb 100644 --- a/triggers/service/auth/src/main/scala/com/daml/auth/middleware/oauth2/RequestTemplates.scala +++ b/triggers/service/auth/src/main/scala/com/daml/auth/middleware/oauth2/RequestTemplates.scala @@ -18,7 +18,7 @@ import scala.util.Try private[oauth2] class RequestTemplates( clientId: String, - clientSecret: String, + clientSecret: SecretString, authTemplate: Option[Path], tokenTemplate: Option[Path], refreshTemplate: Option[Path], @@ -99,6 +99,11 @@ private[oauth2] class RequestTemplates( interpretJsonnet(jsonnet_src, jsonnet_path, args).flatMap(toRequestParams) } + private lazy val config: ujson.Value = ujson.Obj( + "clientId" -> clientId, + "clientSecret" -> clientSecret.value, + ) + private lazy val authJsonnetSource: (String, sjsonnet.Path) = jsonnetSource(authTemplate, authResourcePath) private def authArguments( @@ -107,10 +112,7 @@ private[oauth2] class RequestTemplates( redirectUri: Uri, ): Map[String, ujson.Value] = Map( - "config" -> ujson.Obj( - "clientId" -> clientId, - "clientSecret" -> clientSecret, - ), + "config" -> config, "request" -> ujson.Obj( "claims" -> ujson.Obj( "admin" -> claims.admin, @@ -136,10 +138,7 @@ private[oauth2] class RequestTemplates( private lazy val tokenJsonnetSource: (String, sjsonnet.Path) = jsonnetSource(tokenTemplate, tokenResourcePath) private def tokenArguments(code: String, redirectUri: Uri): Map[String, ujson.Value] = Map( - "config" -> ujson.Obj( - "clientId" -> clientId, - "clientSecret" -> clientSecret, - ), + "config" -> config, "request" -> ujson.Obj( "code" -> code, "redirectUri" -> redirectUri.toString, @@ -151,10 +150,7 @@ private[oauth2] class RequestTemplates( private lazy val refreshJsonnetSource: (String, sjsonnet.Path) = jsonnetSource(refreshTemplate, refreshResourcePath) private def refreshArguments(refreshToken: RefreshToken): Map[String, ujson.Value] = Map( - "config" -> ujson.Obj( - "clientId" -> clientId, - "clientSecret" -> clientSecret, - ), + "config" -> config, "request" -> ujson.Obj( "refreshToken" -> RefreshToken.unwrap(refreshToken) ), @@ -168,7 +164,7 @@ object RequestTemplates { def apply( clientId: String, - clientSecret: String, + clientSecret: SecretString, authTemplate: Option[Path], tokenTemplate: Option[Path], refreshTemplate: Option[Path], diff --git a/triggers/service/auth/src/test/scala/com/daml/auth/middleware/oauth2/TestFixture.scala b/triggers/service/auth/src/test/scala/com/daml/auth/middleware/oauth2/TestFixture.scala index 1d6efb547714..ee9aed70ccc9 100644 --- a/triggers/service/auth/src/test/scala/com/daml/auth/middleware/oauth2/TestFixture.scala +++ b/triggers/service/auth/src/test/scala/com/daml/auth/middleware/oauth2/TestFixture.scala @@ -107,7 +107,7 @@ trait TestFixture oauthTokenTemplate = None, oauthRefreshTemplate = None, clientId = "middleware", - clientSecret = "middleware-secret", + clientSecret = SecretString("middleware-secret"), tokenVerifier = new JwtVerifier( JWT .require(Algorithm.HMAC256(jwtSecret)) diff --git a/triggers/service/auth/src/test/scala/com/daml/auth/middleware/oauth2/TestRequestTemplates.scala b/triggers/service/auth/src/test/scala/com/daml/auth/middleware/oauth2/TestRequestTemplates.scala index 2e4bb74590b4..cd61cfba43bf 100644 --- a/triggers/service/auth/src/test/scala/com/daml/auth/middleware/oauth2/TestRequestTemplates.scala +++ b/triggers/service/auth/src/test/scala/com/daml/auth/middleware/oauth2/TestRequestTemplates.scala @@ -22,7 +22,7 @@ class TestRequestTemplates with TryValues with PartialFunctionValues { private val clientId = "client-id" - private val clientSecret = "client-secret" + private val clientSecret = SecretString("client-secret") private def getTemplates( authTemplate: Option[Path] = None, @@ -191,7 +191,7 @@ class TestRequestTemplates val params = templates.createTokenRequest(code, redirectUri).success.value params shouldBe Map( "client_id" -> clientId, - "client_secret" -> clientSecret, + "client_secret" -> clientSecret.value, "code" -> code, "grant_type" -> "authorization_code", "redirect_uri" -> redirectUri.toString, @@ -205,7 +205,7 @@ class TestRequestTemplates val params = templates.createRefreshRequest(refreshToken).success.value params shouldBe Map( "client_id" -> clientId, - "client_secret" -> clientSecret, + "client_secret" -> clientSecret.value, "grant_type" -> "refresh_code", "refresh_token" -> refreshToken, ) diff --git a/triggers/service/src/test/scala/com/digitalasset/daml/lf/engine/trigger/TriggerServiceFixture.scala b/triggers/service/src/test/scala/com/digitalasset/daml/lf/engine/trigger/TriggerServiceFixture.scala index c9bd0f3fdfda..90d5eb692184 100644 --- a/triggers/service/src/test/scala/com/digitalasset/daml/lf/engine/trigger/TriggerServiceFixture.scala +++ b/triggers/service/src/test/scala/com/digitalasset/daml/lf/engine/trigger/TriggerServiceFixture.scala @@ -19,7 +19,11 @@ import com.auth0.jwt.JWTVerifier.BaseVerification import com.auth0.jwt.algorithms.Algorithm import com.auth0.jwt.interfaces.{Clock => Auth0Clock} import com.daml.auth.middleware.api.{Client => AuthClient} -import com.daml.auth.middleware.oauth2.{Config => MiddlewareConfig, Server => MiddlewareServer} +import com.daml.auth.middleware.oauth2.{ + Config => MiddlewareConfig, + Server => MiddlewareServer, + SecretString, +} import com.daml.auth.oauth2.test.server.{Config => OAuthConfig, Server => OAuthServer} import com.daml.bazeltools.BazelRunfiles import com.daml.clock.AdjustableClock @@ -225,7 +229,7 @@ trait AuthMiddlewareFixture oauthTokenTemplate = None, oauthRefreshTemplate = None, clientId = "oauth-middleware-id", - clientSecret = "oauth-middleware-secret", + clientSecret = SecretString("oauth-middleware-secret"), tokenVerifier = authVerifier, ) middleware <- Resource(MiddlewareServer.start(middlewareConfig))(closeServerBinding)