Merge pull request #5261 from input-output-hk/jordan/cardano-testnet-…

…remove-base-from-conf

Remove base record from Conf

cardano-node-chairman/test/Spec/Chairman/Cardano.hs

@@ -23,7 +23,7 @@ hprop_chairman :: H.Property
 hprop_chairman = H.integrationRetryWorkspace 2 "cardano-chairman" $ \tempAbsPath' -> do
   base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
   configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
-  conf <- H.mkConf (H.ProjectBase base) (Just $ H.YamlFilePath configurationTemplate) tempAbsPath' Nothing
+  conf <- H.mkConf (Just $ H.YamlFilePath configurationTemplate) tempAbsPath' Nothing
 
   allNodes <- fmap H.nodeName . H.allNodes <$> H.testnet (H.CardanoOnlyTestnetOptions H.cardanoDefaultTestnetOptions) conf
 

cardano-node-chairman/testnet/Testnet/Run.hs

@@ -27,7 +27,7 @@ testnetProperty :: Maybe Int -> (H.Conf -> H.Integration ()) -> H.Property
 testnetProperty maybeTestnetMagic tn = H.integrationRetryWorkspace 2 "testnet-chairman" $ \tempAbsPath' -> do
   base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
   configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
-  conf <- H.mkConf (H.ProjectBase base) (Just $ H.YamlFilePath configurationTemplate) tempAbsPath' maybeTestnetMagic
+  conf <- H.mkConf (Just $ H.YamlFilePath configurationTemplate) tempAbsPath' maybeTestnetMagic
 
   -- Fork a thread to keep alive indefinitely any resources allocated by testnet.
   void . liftResourceT . resourceForkIO . forever . liftIO $ IO.threadDelay 10000000

cardano-testnet/cardano-testnet.cabal

@@ -38,6 +38,9 @@ library
                       , cardano-cli
                       , cardano-crypto-class
                       , cardano-crypto-wrapper
+                      , cardano-ledger-alonzo
+                      , cardano-ledger-conway
+                      , cardano-ledger-core
                       , cardano-git-rev
                       , cardano-ledger-core
                       , cardano-ledger-byron
@@ -72,7 +75,10 @@ library
   exposed-modules:      Cardano.Testnet
                         Testnet.Babbage
                         Testnet.Byron
+                        Testnet.Commands.Genesis
                         Testnet.Options
+                        Testnet.Topology 
+                        Testnet.Utils
                         Testnet.Util.Assert
                         Testnet.Util.Base
                         Testnet.Util.Cli
@@ -90,11 +96,9 @@ library
                         Testnet
                         Testnet.Cardano
                         Testnet.Conf
-                        Testnet.Commands.Genesis
                         Testnet.Commands.Governance
                         Testnet.Run
                         Testnet.Shelley
-                        Testnet.Utils
                         Paths_cardano_testnet
 
   autogen-modules:      Paths_cardano_testnet

cardano-testnet/src/Cardano/Testnet.hs

@@ -38,7 +38,7 @@ module Cardano.Testnet (
 
 import           Testnet
 import           Testnet.Cardano
-import           Testnet.Conf hiding (base)
+import           Testnet.Conf
 import           Testnet.Options
 import           Testnet.Shelley as Shelley
 import           Testnet.Utils (waitUntilEpoch)

cardano-testnet/src/Testnet/Babbage.hs

@@ -18,6 +18,7 @@ import           Prelude
 
 import           Control.Monad
 import           Data.Aeson (Value (..), encode, object, toJSON, (.=))
+import           Data.Bifunctor
 import qualified Data.ByteString.Lazy as LBS
 import qualified Data.HashMap.Lazy as HM
 import qualified Data.List as L
@@ -54,7 +55,7 @@ startTimeOffsetSeconds = if OS.isWin32 then 90 else 15
 babbageTestnet :: BabbageTestnetOptions -> H.Conf -> H.Integration TestnetRuntime
 babbageTestnet testnetOptions H.Conf {..} = do
   H.lbsWriteFile (tempAbsPath </> "byron.genesis.spec.json")
-    . encode $ defaultByronGenesisJsonValue
+    . encode $ defaultByronProtocolParamsJsonValue
 
   void $ H.note OS.os
   currentTime <- H.noteShowIO DTC.getCurrentTime
@@ -72,13 +73,12 @@ babbageTestnet testnetOptions H.Conf {..} = do
   -- are deprecated, we must use the "create-staked" cli command to create
   -- SPOs in the ShelleyGenesis
 
-  alonzoBabbageTestGenesisJsonSourceFile <- H.noteShow $ base </> "scripts/babbage/alonzo-babbage-test-genesis.json"
   alonzoBabbageTestGenesisJsonTargetFile <- H.noteShow $ tempAbsPath </> "genesis.alonzo.spec.json"
-  H.copyFile alonzoBabbageTestGenesisJsonSourceFile alonzoBabbageTestGenesisJsonTargetFile
+  gen <- H.evalEither $ first displayError defaultAlonzoGenesis
+  H.evalIO $ LBS.writeFile alonzoBabbageTestGenesisJsonTargetFile $ encode gen
 
-  conwayBabbageTestGenesisJsonSourceFile <- H.noteShow $ base </> "scripts/babbage/conway-babbage-test-genesis.json"
   conwayBabbageTestGenesisJsonTargetFile <- H.noteShow $ tempAbsPath </> "genesis.conway.spec.json"
-  H.copyFile conwayBabbageTestGenesisJsonSourceFile conwayBabbageTestGenesisJsonTargetFile
+  H.evalIO $ LBS.writeFile conwayBabbageTestGenesisJsonTargetFile $ encode defaultConwayGenesis
 
   configurationFile <- H.noteShow $ tempAbsPath </> "configuration.yaml"
 

cardano-testnet/src/Testnet/Byron.hs

@@ -1,5 +1,4 @@
 {-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE RecordWildCards #-}
 {-# LANGUAGE TypeApplications #-}
 
 {-# OPTIONS_GHC -Wno-unused-local-binds -Wno-unused-matches #-}
@@ -14,6 +13,7 @@ module Testnet.Byron
 import           Control.Monad (forM_, void, when)
 import           Data.Aeson (Value)
 import           Data.ByteString.Lazy (ByteString)
+import qualified Data.ByteString.Lazy as LBS
 import           Data.Functor ((<&>))
 import           Hedgehog.Extras.Stock.Aeson (rewriteObject)
 import           Hedgehog.Extras.Stock.IO.Network.Sprocket (Sprocket (..))
@@ -22,12 +22,13 @@ import           Ouroboros.Network.PeerSelection.LedgerPeers (UseLedgerAfter (..
 import           Ouroboros.Network.PeerSelection.RelayAccessPoint (RelayAccessPoint (..))
 import           System.FilePath.Posix ((</>))
 
+import           Cardano.Api hiding (Value)
+
 import qualified Cardano.Node.Configuration.Topology as NonP2P
 import qualified Cardano.Node.Configuration.TopologyP2P as P2P
 import qualified Data.Aeson as J
 import qualified Data.HashMap.Lazy as HM
 import qualified Data.List as L
-import qualified Data.Text as T
 import qualified Data.Time.Clock as DTC
 import qualified Hedgehog as H
 import qualified Hedgehog.Extras.Stock.IO.File as IO
@@ -41,8 +42,11 @@ import qualified Hedgehog.Extras.Test.Process as H
 import qualified System.Info as OS
 import qualified System.IO as IO
 import qualified System.Process as IO
+import           Testnet.Commands.Genesis
 import qualified Testnet.Conf as H
+import           Testnet.Options hiding (defaultTestnetOptions)
 import qualified Testnet.Util.Process as H
+import           Testnet.Utils
 
 {- HLINT ignore "Reduce duplication" -}
 {- HLINT ignore "Redundant <&>" -}
@@ -68,42 +72,28 @@ defaultTestnetOptions = TestnetOptions
   , enableP2P = False
   }
 
-replaceNodeLog :: Int -> String -> String
-replaceNodeLog n s = T.unpack (T.replace "logs/node-0.log" replacement (T.pack s))
-  where replacement = T.pack ("logs/node-" <> show @Int n <> ".log")
-
--- TODO: We need to refactor this to directly check the parsed configuration
--- and fail with a suitable error message.
--- | Rewrite a line in the configuration file
-rewriteConfiguration :: Bool -> Int -> String -> String
-rewriteConfiguration _ _ "TraceBlockchainTime: False"          = "TraceBlockchainTime: True"
-rewriteConfiguration _ n s | "logs/node-0.log" `L.isInfixOf` s = replaceNodeLog n s
-rewriteConfiguration True _ "EnableP2P: False"                 = "EnableP2P: True"
-rewriteConfiguration False _ "EnableP2P: True"                 = "EnableP2P: False"
-rewriteConfiguration _ _ s                                     = s
-
 rewriteParams :: TestnetOptions -> Value -> Value
 rewriteParams testnetOptions = rewriteObject
   $ HM.insert "slotDuration" (J.toJSON @String (show @Int (slotDuration testnetOptions)))
 
 mkTopologyConfig :: Int -> Int -> [Int]
                  -> Bool -- ^ if true use p2p topology configuration
                  -> ByteString
-mkTopologyConfig i numBftNodes allPorts False = J.encode topologyNonP2P
+mkTopologyConfig i numBftNodes' allPorts False = J.encode topologyNonP2P
   where
     topologyNonP2P :: NonP2P.NetworkTopology
     topologyNonP2P =
       NonP2P.RealNodeTopology
-        $ flip fmap ([0 .. numBftNodes - 1] L.\\ [i])
+        $ flip fmap ([0 .. numBftNodes' - 1] L.\\ [i])
         $ \j -> NonP2P.RemoteAddress "127.0.0.1"
                                     (fromIntegral $ allPorts L.!! j)
                                     1
-mkTopologyConfig i numBftNodes allPorts True = J.encode topologyP2P
+mkTopologyConfig i numBftNodes' allPorts True = J.encode topologyP2P
   where
     rootConfig :: P2P.RootConfig
     rootConfig =
       P2P.RootConfig
-        (flip fmap ([0 .. numBftNodes - 1] L.\\ [i])
+        (flip fmap ([0 .. numBftNodes' - 1] L.\\ [i])
         $ \j -> RelayAccessAddress "127.0.0.1"
                             (fromIntegral $ allPorts L.!! j)
         )
@@ -113,7 +103,7 @@ mkTopologyConfig i numBftNodes allPorts True = J.encode topologyP2P
     localRootPeerGroups =
       P2P.LocalRootPeersGroups
         [ P2P.LocalRootPeersGroup rootConfig
-                                  (numBftNodes - 1)
+                                  (numBftNodes' - 1)
         ]
 
     topologyP2P :: P2P.NetworkTopology
@@ -125,28 +115,34 @@ mkTopologyConfig i numBftNodes allPorts True = J.encode topologyP2P
 
 
 testnet :: TestnetOptions -> H.Conf -> H.Integration [String]
-testnet testnetOptions H.Conf {..} = do
+testnet testnetOptions conf = do
   void $ H.note OS.os
-  baseConfig <- H.noteShow $ base </> "configuration/chairman/defaults/simpleview"
+  let tNetMagic = H.testnetMagic conf
   currentTime <- H.noteShowIO DTC.getCurrentTime
   startTime <- H.noteShow $ DTC.addUTCTime 15 currentTime -- 15 seconds into the future
   allPorts <- H.noteShowIO $ IO.allocateRandomPorts (numBftNodes testnetOptions)
+  let tempAbsPath' = H.tempAbsPath conf
+      sockDir = H.socketDir conf
+      tempBaseAbsPath' = H.tempBaseAbsPath conf
+
+  H.lbsWriteFile (tempAbsPath' </> "byron.genesis.spec.json")
+    . J.encode $ defaultByronProtocolParamsJsonValue
 
   H.copyRewriteJsonFile
-    (base </> "scripts/protocol-params.json")
-    (tempAbsPath </> "protocol-params.json")
+    (tempAbsPath' </> "byron.genesis.spec.json")
+    (tempAbsPath' </> "protocol-params.json")
     (rewriteParams testnetOptions)
 
   -- Generate keys
   void $ H.execCli
     [ "byron"
     , "genesis"
     , "genesis"
-    , "--genesis-output-dir", tempAbsPath </> "genesis"
+    , "--genesis-output-dir", tempAbsPath' </> "genesis"
     , "--start-time", showUTCTimeSeconds startTime
-    , "--protocol-parameters-file", tempAbsPath </> "protocol-params.json"
+    , "--protocol-parameters-file", tempAbsPath' </> "protocol-params.json"
     , "--k", show @Int (securityParam testnetOptions)
-    , "--protocol-magic", show @Int testnetMagic
+    , "--protocol-magic", show @Int tNetMagic
     , "--n-poor-addresses", show @Int (nPoorAddresses testnetOptions)
     , "--n-delegate-addresses", show @Int (numBftNodes testnetOptions)
     , "--total-balance", show @Int (totalBalance testnetOptions)
@@ -156,10 +152,10 @@ testnet testnetOptions H.Conf {..} = do
     , "--secret-seed", "2718281828"
     ]
 
-  H.writeFile (tempAbsPath </> "genesis/GENHASH") . S.lastLine =<< H.execCli
+  H.writeFile (tempAbsPath' </> "genesis/GENHASH") . S.lastLine =<< H.execCli
     [ "print-genesis-hash"
     , "--genesis-json"
-    , tempAbsPath </> "genesis/genesis.json"
+    , tempAbsPath' </> "genesis/genesis.json"
     ]
 
   let nodeIndexes = [0..numBftNodes testnetOptions - 1]
@@ -168,22 +164,29 @@ testnet testnetOptions H.Conf {..} = do
   -- Launch cluster of three nodes in P2P Mode
   forM_ nodeIndexes $ \i -> do
     si <- H.noteShow $ show @Int i
-    nodeStdoutFile <- H.noteTempFile tempAbsPath $ "cardano-node-" <> si <> ".stdout.log"
-    nodeStderrFile <- H.noteTempFile tempAbsPath $ "cardano-node-" <> si <> ".stderr.log"
-    sprocket <- H.noteShow $ Sprocket tempBaseAbsPath (socketDir </> "node-" <> si)
+    nodeStdoutFile <- H.noteTempFile tempAbsPath' $ "cardano-node-" <> si <> ".stdout.log"
+    nodeStderrFile <- H.noteTempFile tempAbsPath' $ "cardano-node-" <> si <> ".stderr.log"
+    sprocket <- H.noteShow $ Sprocket tempBaseAbsPath' (sockDir </> "node-" <> si)
     portString <- H.note $ show @Int (allPorts L.!! i)
-    topologyFile <- H.noteShow $ tempAbsPath </> "topology-node-" <> si <> ".json"
-    configFile <- H.noteShow $ tempAbsPath </> "config-" <> si <> ".yaml"
-    signingKeyFile <- H.noteShow $ tempAbsPath </> "genesis/delegate-keys.00" <> si <> ".key"
-    delegationCertificateFile <- H.noteShow $ tempAbsPath </> "genesis/delegation-cert.00" <> si <> ".json"
+    topologyFile <- H.noteShow $ tempAbsPath' </> "topology-node-" <> si <> ".json"
+    configFile <- H.noteShow $ tempAbsPath' </> "config-" <> si <> ".yaml"
+    signingKeyFile <- H.noteShow $ tempAbsPath' </> "genesis/delegate-keys.00" <> si <> ".key"
+    delegationCertificateFile <- H.noteShow $ tempAbsPath' </> "genesis/delegation-cert.00" <> si <> ".json"
 
-    dbDir <- H.createDirectoryIfMissing $ tempAbsPath </> "db/node-" <> si
+    dbDir <- H.createDirectoryIfMissing $ tempAbsPath' </> "db/node-" <> si
 
-    H.lbsWriteFile (tempAbsPath </> "topology-node-" <> si <> ".json") $
+    H.lbsWriteFile (tempAbsPath' </> "topology-node-" <> si <> ".json") $
       mkTopologyConfig i (numBftNodes testnetOptions) allPorts (enableP2P testnetOptions)
 
-    H.writeFile (tempAbsPath </> "config-" <> si <> ".yaml") . L.unlines . fmap (rewriteConfiguration (enableP2P testnetOptions) i) . L.lines =<<
-      H.readFile (baseConfig </> "config-0.yaml")
+    byronGenesisHash <- getByronGenesisHash $ tempAbsPath' </> "genesis/genesis.json"
+
+    let finalYamlConfig :: LBS.ByteString
+        finalYamlConfig = J.encode . J.Object
+                            $ mconcat [ byronGenesisHash
+                                      , defaultYamlHardforkViaConfig $ AnyCardanoEra ByronEra
+                                      ]
+
+    H.evalIO $ LBS.writeFile (tempAbsPath' </> "config-" <> si <> ".yaml")  finalYamlConfig
 
     hNodeStdout <- H.openFile nodeStdoutFile IO.WriteMode
     hNodeStderr <- H.openFile nodeStderrFile IO.WriteMode
@@ -206,7 +209,7 @@ testnet testnetOptions H.Conf {..} = do
           { IO.std_in = IO.CreatePipe
           , IO.std_out = IO.UseHandle hNodeStdout
           , IO.std_err = IO.UseHandle hNodeStderr
-          , IO.cwd = Just tempBaseAbsPath
+          , IO.cwd = Just tempBaseAbsPath'
           }
         )
       )
@@ -219,16 +222,16 @@ testnet testnetOptions H.Conf {..} = do
 
   forM_ nodeIndexes $ \i -> do
     si <- H.noteShow $ show @Int i
-    sprocket <- H.noteShow $ Sprocket tempBaseAbsPath (socketDir </> "node-" <> si)
+    sprocket <- H.noteShow $ Sprocket tempBaseAbsPath' (sockDir </> "node-" <> si)
     _spocketSystemNameFile <- H.noteShow $ IO.sprocketSystemName sprocket
     -- TODO: Better error message need to indicate a sprocket was not created
     H.byDeadlineM 10 deadline "Failed to connect to node socket" $ H.assertM $ H.doesSprocketExist sprocket
 
   forM_ nodeIndexes $ \i -> do
     si <- H.noteShow $ show @Int i
-    nodeStdoutFile <- H.noteTempFile tempAbsPath $ "cardano-node-" <> si <> ".stdout.log"
+    nodeStdoutFile <- H.noteTempFile tempAbsPath' $ "cardano-node-" <> si <> ".stdout.log"
     H.assertByDeadlineIOCustom "stdout does not contain \"until genesis start time\"" deadline $ IO.fileContains "until genesis start time at" nodeStdoutFile
 
-  H.copyFile (tempAbsPath </> "config-1.yaml") (tempAbsPath </> "configuration.yaml")
+  H.copyFile (tempAbsPath' </> "config-1.yaml") (tempAbsPath' </> "configuration.yaml")
 
   return allNodes

cardano-testnet/src/Testnet/Cardano.hs

@@ -26,6 +26,7 @@ import           Cardano.Api hiding (cardanoEra)
 
 import           Control.Monad
 import qualified Data.Aeson as J
+import           Data.Bifunctor
 import qualified Data.ByteString.Lazy as LBS
 import qualified Data.HashMap.Lazy as HM
 import           Data.List ((\\))
@@ -219,7 +220,7 @@ cardanoTestnet testnetOptions H.Conf {..} = do
     H.writeFile (tempAbsPath </> node </> "port") (show port)
 
   H.lbsWriteFile (tempAbsPath </> "byron.genesis.spec.json")
-    . J.encode $ defaultByronGenesisJsonValue
+    . J.encode $ defaultByronProtocolParamsJsonValue
 
   -- stuff
   execCli_
@@ -317,15 +318,13 @@ cardanoTestnet testnetOptions H.Conf {..} = do
   -- Set up our template
   shelleyDir <- H.createDirectoryIfMissing $ tempAbsPath </> "shelley"
 
-  -- TODO: This is fragile, we should be passing in all necessary
-  -- configuration files.
-  let sourceAlonzoGenesisSpecFile = base </> "cardano-testnet/files/data/alonzo/genesis.alonzo.spec.json"
   alonzoSpecFile <- H.noteTempFile tempAbsPath "shelley/genesis.alonzo.spec.json"
-  H.copyFile sourceAlonzoGenesisSpecFile alonzoSpecFile
+  gen <- H.evalEither $ first displayError defaultAlonzoGenesis
+  H.evalIO $ LBS.writeFile alonzoSpecFile $ J.encode gen
+
 
-  let sourceConwayGenesisSpecFile = base </> "cardano-testnet/files/data/conway/genesis.conway.spec.json"
   conwaySpecFile <- H.noteTempFile tempAbsPath "shelley/genesis.conway.spec.json"
-  H.copyFile sourceConwayGenesisSpecFile conwaySpecFile
+  H.evalIO $ LBS.writeFile conwaySpecFile $ J.encode defaultConwayGenesis
 
   execCli_
     [ "genesis", "create"