Merge #3727

3727: Make configuration template in cardano-testnet configurable r=newhoggy a=newhoggy

The change is for this: IntersectMBO/plutus-apps#364 (comment)

Co-authored-by: John Ky <john.ky@iohk.io>

cardano-cli/test/Test/Config/Mainnet.hs

@@ -29,20 +29,20 @@ import qualified System.Directory as IO
 
 hprop_configMainnetHash :: Property
 hprop_configMainnetHash = H.propertyOnce $ do
-  projectBase <- H.note =<< H.evalIO . IO.canonicalizePath =<< H.getProjectBase
-  result <- H.evalIO $ runExceptT $ initialLedgerState $ projectBase </> "configuration/cardano/mainnet-config.json"
+  base <- H.note =<< H.evalIO . IO.canonicalizePath =<< H.getProjectBase
+  result <- H.evalIO $ runExceptT $ initialLedgerState $ base </> "configuration/cardano/mainnet-config.json"
   case result of
     Right (_, _) -> return ()
     Left e -> H.failWithCustom GHC.callStack Nothing (T.unpack (renderInitialLedgerStateError e))
 
 hprop_configMainnetYaml :: Property
 hprop_configMainnetYaml = H.propertyOnce $ do
-  projectBase <- H.note =<< H.evalIO . IO.canonicalizePath =<< H.getProjectBase
-  yamlResult <- H.evalIO . Y.decodeFileEither $ projectBase </> "configuration/cardano/mainnet-config.yaml"
+  base <- H.note =<< H.evalIO . IO.canonicalizePath =<< H.getProjectBase
+  yamlResult <- H.evalIO . Y.decodeFileEither $ base </> "configuration/cardano/mainnet-config.yaml"
   yaml :: J.Value <- case yamlResult of
     Right v -> return v
     Left e -> H.failWithCustom GHC.callStack Nothing (Y.prettyPrintParseException e)
-  jsonResult <- H.evalIO . J.eitherDecodeFileStrict $ projectBase </> "configuration/cardano/mainnet-config.json"
+  jsonResult <- H.evalIO . J.eitherDecodeFileStrict $ base </> "configuration/cardano/mainnet-config.json"
   json  :: J.Value <- case jsonResult of
     Right v -> return v
     Left e -> H.failWithCustom GHC.callStack Nothing (show e)

cardano-node-chairman/cardano-node-chairman.cabal

@@ -63,6 +63,7 @@ test-suite chairman-tests
   type:                 exitcode-stdio-1.0
 
   build-depends:        cardano-testnet
+                      , directory
                       , filepath
                       , hedgehog
                       , hedgehog-extras

cardano-node-chairman/test/Spec/Chairman/Byron.hs

@@ -4,12 +4,16 @@ module Spec.Chairman.Byron
   ( hprop_chairman
   ) where
 
+import           Control.Monad ((=<<))
 import           Data.Function
 import           Data.Maybe
 import           Spec.Chairman.Chairman (chairmanOver)
+import           System.FilePath ((</>))
 
 import qualified Hedgehog as H
 import qualified Hedgehog.Extras.Test.Base as H
+import qualified Hedgehog.Extras.Test.Process as H
+import qualified System.Directory as IO
 import qualified Test.Base as H
 import qualified Testnet.Byron as H
 import qualified Testnet.Conf as H
@@ -20,7 +24,9 @@ import qualified Testnet.Conf as H
 
 hprop_chairman :: H.Property
 hprop_chairman = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAbsPath' -> do
-  conf <- H.mkConf tempAbsPath' Nothing
+  base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
+  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
+  conf <- H.mkConf (H.ProjectBase base) (H.YamlFilePath configurationTemplate) tempAbsPath' Nothing
   allNodes <- H.testnet H.defaultTestnetOptions conf
 
   chairmanOver 120 52 conf allNodes

cardano-node-chairman/test/Spec/Chairman/Cardano.hs

@@ -4,13 +4,17 @@ module Spec.Chairman.Cardano
   ( hprop_chairman
   ) where
 
+import           Control.Monad ((=<<))
 import           Data.Function
 import           Data.Functor
 import           Data.Maybe
 import           Spec.Chairman.Chairman (chairmanOver)
+import           System.FilePath ((</>))
 
 import qualified Hedgehog as H
 import qualified Hedgehog.Extras.Test.Base as H
+import qualified Hedgehog.Extras.Test.Process as H
+import qualified System.Directory as IO
 import qualified Test.Base as H
 import qualified Testnet.Cardano as H
 import qualified Testnet.Conf as H
@@ -21,7 +25,9 @@ import qualified Testnet.Conf as H
 
 hprop_chairman :: H.Property
 hprop_chairman = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAbsPath' -> do
-  conf <- H.mkConf tempAbsPath' Nothing
+  base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
+  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
+  conf <- H.mkConf (H.ProjectBase base) (H.YamlFilePath configurationTemplate) tempAbsPath' Nothing
 
   allNodes <- fmap H.nodeName . H.allNodes <$> H.testnet H.defaultTestnetOptions conf
 

cardano-node-chairman/test/Spec/Chairman/Shelley.hs

@@ -4,12 +4,16 @@ module Spec.Chairman.Shelley
   ( hprop_chairman
   ) where
 
+import           Control.Monad ((=<<))
 import           Data.Function
 import           Data.Maybe
 import           Spec.Chairman.Chairman (chairmanOver)
+import           System.FilePath ((</>))
 
 import qualified Hedgehog as H
 import qualified Hedgehog.Extras.Test.Base as H
+import qualified Hedgehog.Extras.Test.Process as H
+import qualified System.Directory as IO
 import qualified Test.Base as H
 import qualified Testnet.Conf as H
 import qualified Testnet.Shelley as H
@@ -20,7 +24,9 @@ import qualified Testnet.Shelley as H
 
 hprop_chairman :: H.Property
 hprop_chairman = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAbsPath' -> do
-  conf <- H.mkConf tempAbsPath' Nothing
+  base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
+  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
+  conf <- H.mkConf (H.ProjectBase base) (H.YamlFilePath configurationTemplate) tempAbsPath' Nothing
 
   allNodes <- H.testnet H.defaultTestnetOptions conf
 

cardano-node-chairman/testnet/Testnet/Run.hs

@@ -16,6 +16,7 @@ import qualified Control.Concurrent as IO
 import qualified Control.Concurrent.STM as STM
 import qualified Hedgehog as H
 import qualified Hedgehog.Extras.Test.Base as H
+import qualified Hedgehog.Extras.Test.Process as H
 import qualified System.Console.ANSI as ANSI
 import qualified System.Exit as IO
 import qualified System.IO as IO
@@ -24,7 +25,9 @@ import qualified Testnet.Conf as H
 
 testnetProperty :: Maybe Int -> (H.Conf -> H.Integration ()) -> H.Property
 testnetProperty maybeTestnetMagic tn = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAbsPath' -> do
-  conf <- H.mkConf tempAbsPath' maybeTestnetMagic
+  base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
+  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
+  conf <- H.mkConf (H.ProjectBase base) (H.YamlFilePath configurationTemplate) tempAbsPath' maybeTestnetMagic
 
   -- Fork a thread to keep alive indefinitely any resources allocated by testnet.
   void . liftResourceT . resourceForkIO . forever . liftIO $ IO.threadDelay 10000000

cardano-testnet/cardano-testnet.cabal

@@ -78,6 +78,8 @@ executable cardano-testnet
   build-depends:        ansi-terminal
                       , cardano-git-rev
                       , cardano-testnet
+                      , directory
+                      , filepath
                       , hedgehog
                       , hedgehog-extras
                       , optparse-applicative-fork

cardano-testnet/src/Testnet/Cardano.hs

@@ -222,7 +222,6 @@ testnet testnetOptions H.Conf {..} = do
   void $ H.note OS.os
   currentTime <- H.noteShowIO DTC.getCurrentTime
   startTime <- H.noteShow $ DTC.addUTCTime startTimeOffsetSeconds currentTime
-  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
   configurationFile <- H.noteShow $ tempAbsPath </> "configuration.yaml"
   let numBftNodes = length (bftNodeOptions testnetOptions)
       bftNodesN = [1 .. numBftNodes]

cardano-testnet/src/Testnet/Conf.hs

@@ -2,7 +2,9 @@
 {-# LANGUAGE RecordWildCards #-}
 
 module Testnet.Conf
-  ( Conf(..)
+  ( ProjectBase(..)
+  , YamlFilePath(..)
+  , Conf(..)
   , mkConf
 
   , SubmitApiConfig(..)
@@ -19,26 +21,33 @@ import           System.IO (FilePath)
 import           Text.Show
 
 import qualified Hedgehog.Extras.Test.Base as H
-import qualified Hedgehog.Extras.Test.Process as H
 import qualified System.FilePath.Posix as FP
 import qualified System.Random as IO
 
+newtype ProjectBase = ProjectBase
+  { projectBase :: FilePath
+  } deriving (Eq, Show)
+
+newtype YamlFilePath = YamlFilePath
+  { projectBase :: FilePath
+  } deriving (Eq, Show)
+
 data Conf = Conf
   { tempAbsPath :: FilePath
   , tempRelPath :: FilePath
   , tempBaseAbsPath :: FilePath
   , logDir :: FilePath
   , base :: FilePath
   , socketDir :: FilePath
+  , configurationTemplate :: FilePath
   , testnetMagic :: Int
   } deriving (Eq, Show)
 
-mkConf :: FilePath -> Maybe Int -> H.Integration Conf
-mkConf tempAbsPath maybeMagic = do
+mkConf :: ProjectBase -> YamlFilePath -> FilePath -> Maybe Int -> H.Integration Conf
+mkConf (ProjectBase base) (YamlFilePath configurationTemplate) tempAbsPath maybeMagic = do
   testnetMagic <- H.noteShowIO $ maybe (IO.randomRIO (1000, 2000)) return maybeMagic
   tempBaseAbsPath <- H.noteShow $ FP.takeDirectory tempAbsPath
   tempRelPath <- H.noteShow $ FP.makeRelative tempBaseAbsPath tempAbsPath
-  base <- H.noteShowM H.getProjectBase
   socketDir <- H.noteShow $ tempRelPath </> "socket"
   logDir <- H.noteTempFile tempAbsPath "logs"
 

cardano-testnet/src/Testnet/Shelley.hs

@@ -485,7 +485,9 @@ testnet testnetOptions H.Conf {..} = do
 
 hprop_testnet :: H.Property
 hprop_testnet = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAbsPath' -> do
-  conf <- H.mkConf tempAbsPath' Nothing
+  base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
+  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
+  conf <- H.mkConf (H.ProjectBase base) (H.YamlFilePath configurationTemplate) tempAbsPath' Nothing
 
   void . liftResourceT . resourceForkIO . forever . liftIO $ IO.threadDelay 10000000
 

cardano-testnet/test/Spec/Plutus/Direct/CertifyingAndWithdrawingPlutus.hs

@@ -13,7 +13,6 @@ import           Prelude
 
 import           Cardano.Api
 import           Cardano.Api.Shelley
-
 import           Control.Monad (void)
 import qualified Data.Aeson as J
 import qualified Data.Map.Strict as Map
@@ -62,8 +61,10 @@ isLinux = os == "linux"
 hprop_plutus_certifying_withdrawing :: Property
 hprop_plutus_certifying_withdrawing = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAbsBasePath' -> do
   H.note_ SYS.os
-  projectBase <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
-  conf@H.Conf { H.tempBaseAbsPath, H.tempAbsPath } <- H.noteShowM $ H.mkConf tempAbsBasePath' Nothing
+  base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
+  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
+  conf@H.Conf { H.tempBaseAbsPath, H.tempAbsPath } <- H.noteShowM $
+    H.mkConf (H.ProjectBase base) (H.YamlFilePath configurationTemplate) tempAbsBasePath' Nothing
 
   let fastTestnetOptions = defaultTestnetOptions
                              { epochLength = 500
@@ -85,7 +86,6 @@ hprop_plutus_certifying_withdrawing = H.integration . H.runFinallies . H.workspa
         }
 
   -- First we note all the relevant files
-  base <- H.note projectBase
   work <- H.note tempAbsPath
 
   -- We get our UTxOs from here

cardano-testnet/test/Spec/Plutus/Direct/ScriptContextEquality.hs

@@ -60,8 +60,10 @@ instance FromJSON Utxo where
 
 hprop_plutus_script_context_equality :: Property
 hprop_plutus_script_context_equality = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAbsBasePath' -> do
-  projectBase <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
-  conf@H.Conf { H.tempBaseAbsPath, H.tempAbsPath } <- H.noteShowM $ H.mkConf tempAbsBasePath' Nothing
+  base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
+  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
+  conf@H.Conf { H.tempBaseAbsPath, H.tempAbsPath } <- H.noteShowM $
+    H.mkConf (H.ProjectBase base) (H.YamlFilePath configurationTemplate) tempAbsBasePath' Nothing
 
   TC.TestnetRuntime { bftSprockets, testnetMagic } <- testnet defaultTestnetOptions conf
 
@@ -78,7 +80,6 @@ hprop_plutus_script_context_equality = H.integration . H.runFinallies . H.worksp
         }
 
   -- First we note all the relevant files
-  base <- H.note projectBase
   work <- H.note tempAbsPath
 
   -- We get our UTxOs from here

cardano-testnet/test/Spec/Plutus/Direct/ScriptContextEqualityMint.hs

@@ -11,7 +11,6 @@ module Spec.Plutus.Direct.ScriptContextEqualityMint
 import           Prelude
 
 import           Cardano.Api
-
 import           Control.Monad
 import qualified Data.Aeson as J
 import qualified Data.Aeson.Types as Aeson
@@ -48,8 +47,10 @@ millarCoin = BSC.unpack $ Base16.encode "MillarCoin"
 
 hprop_plutus_script_context_mint_equality :: Property
 hprop_plutus_script_context_mint_equality = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAbsBasePath' -> do
-  projectBase <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
-  conf@H.Conf { H.tempBaseAbsPath, H.tempAbsPath } <- H.noteShowM $ H.mkConf tempAbsBasePath' Nothing
+  base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
+  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
+  conf@H.Conf { H.tempBaseAbsPath, H.tempAbsPath } <- H.noteShowM $
+    H.mkConf (H.ProjectBase base) (H.YamlFilePath configurationTemplate) tempAbsBasePath' Nothing
 
   TC.TestnetRuntime { bftSprockets, testnetMagic } <- testnet defaultTestnetOptions conf
 
@@ -66,7 +67,6 @@ hprop_plutus_script_context_mint_equality = H.integration . H.runFinallies . H.w
         }
 
   -- First we note all the relevant files
-  base <- H.note projectBase
   work <- H.note tempAbsPath
 
   -- We get our UTxOs from here

cardano-testnet/test/Spec/Plutus/Direct/TxInLockingPlutus.hs

@@ -61,8 +61,10 @@ instance FromJSON Utxo where
 
 hprop_plutus :: Property
 hprop_plutus = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAbsBasePath' -> do
-  projectBase <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
-  conf@H.Conf { H.tempBaseAbsPath, H.tempAbsPath } <- H.noteShowM $ H.mkConf tempAbsBasePath' Nothing
+  base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
+  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
+  conf@H.Conf { H.tempBaseAbsPath, H.tempAbsPath } <- H.noteShowM $
+    H.mkConf (H.ProjectBase base) (H.YamlFilePath configurationTemplate) tempAbsBasePath' Nothing
 
   H.TestnetRuntime { H.bftSprockets, H.testnetMagic } <- H.testnet H.defaultTestnetOptions conf
 
@@ -78,7 +80,6 @@ hprop_plutus = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAb
         , H.execConfigCwd = Last $ Just tempBaseAbsPath
         }
 
-  base <- H.note projectBase
   work <- H.note tempAbsPath
   utxoVKeyFile <- H.note $ tempAbsPath </> "shelley/utxo-keys/utxo1.vkey"
   utxoSKeyFile <- H.note $ tempAbsPath </> "shelley/utxo-keys/utxo1.skey"

cardano-testnet/test/Spec/Plutus/Script/TxInLockingPlutus.hs

@@ -36,8 +36,10 @@ import qualified Testnet.Conf as H
 
 hprop_plutus :: Property
 hprop_plutus = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAbsBasePath' -> do
-  projectBase <- H.note =<< H.evalIO . IO.canonicalizePath =<< H.getProjectBase
-  conf@H.Conf { H.tempBaseAbsPath, H.tempAbsPath } <- H.noteShowM $ H.mkConf tempAbsBasePath' Nothing
+  base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
+  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
+  conf@H.Conf { H.tempBaseAbsPath, H.tempAbsPath } <- H.noteShowM $
+    H.mkConf (H.ProjectBase base) (H.YamlFilePath configurationTemplate) tempAbsBasePath' Nothing
 
   resultFile <- H.noteTempFile tempAbsPath "result.out"
 
@@ -50,7 +52,7 @@ hprop_plutus = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAb
   let execConfig = H.ExecConfig
         { H.execConfigEnv = Last $ Just
           [ ("CARDANO_CLI", cardanoCli)
-          , ("BASE", projectBase)
+          , ("BASE", base)
           , ("WORK", tempAbsPath)
           , ("UTXO_VKEY", tempAbsPath </> "shelley/utxo-keys/utxo1.vkey")
           , ("UTXO_SKEY", tempAbsPath </> "shelley/utxo-keys/utxo1.skey")
@@ -62,7 +64,7 @@ hprop_plutus = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAb
         , H.execConfigCwd = Last $ Just tempBaseAbsPath
         }
 
-  scriptPath <- H.eval $ projectBase </> "scripts/plutus/example-txin-locking-plutus-script.sh"
+  scriptPath <- H.eval $ base </> "scripts/plutus/example-txin-locking-plutus-script.sh"
 
   H.exec_ execConfig H.bashPath
     [ "-x"

cardano-testnet/test/Spec/Plutus/SubmitApi/TxInLockingPlutus.hs

@@ -62,7 +62,7 @@ instance FromJSON Utxo where
 
 hprop_plutus :: Property
 hprop_plutus = Test.integration . HE.runFinallies . HE.workspace "chairman" $ \tempAbsBasePath' -> do
-  projectBase <- HE.note =<< HE.noteIO . IO.canonicalizePath =<< HE.getProjectBase
+  base <- HE.note =<< HE.noteIO . IO.canonicalizePath =<< HE.getProjectBase
   conf@TN.Conf { TN.tempBaseAbsPath, TN.tempAbsPath } <- HE.noteShowM $ TN.mkConf tempAbsBasePath' Nothing
 
   TN.TestnetRuntime { TN.configurationFile, TN.bftSprockets, TN.testnetMagic } <- TN.testnet TN.defaultTestnetOptions conf
@@ -79,7 +79,6 @@ hprop_plutus = Test.integration . HE.runFinallies . HE.workspace "chairman" $ \t
         , H.execConfigCwd = Last $ Just tempBaseAbsPath
         }
 
-  base <- HE.note projectBase
   work <- HE.note tempAbsPath
   utxoVKeyFile <- HE.note $ tempAbsPath </> "shelley/utxo-keys/utxo1.vkey"
   utxoSKeyFile <- HE.note $ tempAbsPath </> "shelley/utxo-keys/utxo1.skey"

cardano-testnet/test/Spec/Shutdown.hs

@@ -41,8 +41,10 @@ import qualified Testnet.Conf as H
 
 hprop_shutdown :: Property
 hprop_shutdown = H.integration . H.runFinallies . H.workspace "chairman" $ \tempAbsBasePath' -> do
-  projectBase <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
-  H.Conf { H.tempBaseAbsPath, H.tempAbsPath, H.logDir, H.socketDir } <- H.noteShowM $ H.mkConf tempAbsBasePath' Nothing
+  base <- H.note =<< H.noteIO . IO.canonicalizePath =<< H.getProjectBase
+  configurationTemplate <- H.noteShow $ base </> "configuration/defaults/byron-mainnet/configuration.yaml"
+  H.Conf { H.tempBaseAbsPath, H.tempAbsPath, H.logDir, H.socketDir } <- H.noteShowM $
+    H.mkConf (H.ProjectBase base) (H.YamlFilePath configurationTemplate) tempAbsBasePath' Nothing
 
   [port] <- H.noteShowIO $ IO.allocateRandomPorts 1
 
@@ -64,8 +66,8 @@ hprop_shutdown = H.integration . H.runFinallies . H.workspace "chairman" $ \temp
   (mStdin, _mStdout, _mStderr, pHandle, _releaseKey) <- H.createProcess =<<
     ( H.procNode
       [ "run"
-      , "--config", projectBase </> "configuration/cardano/mainnet-config.json"
-      , "--topology", projectBase </> "configuration/cardano/mainnet-topology.json"
+      , "--config", base </> "configuration/cardano/mainnet-config.json"
+      , "--topology", base </> "configuration/cardano/mainnet-topology.json"
       , "--database-path", tempAbsPath </> "db"
       , "--socket-path", IO.sprocketArgumentName sprocket
       , "--host-addr", "127.0.0.1"