Introduce system tests - initial version from wasmd

simapp/simd/cmd/testnet.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"time"
 
 	cmtconfig "github.com/cometbft/cometbft/config"
 	cmttime "github.com/cometbft/cometbft/types/time"
@@ -49,6 +50,8 @@ var (
 	flagRPCAddress        = "rpc.address"
 	flagAPIAddress        = "api.address"
 	flagPrintMnemonic     = "print-mnemonic"
+	flagCommitTimeout     = "commit-timeout"
+	flagSingleHost        = "single-host"
 )
 
 type initArgs struct {
@@ -62,6 +65,7 @@ type initArgs struct {
 	outputDir         string
 	startingIPAddress string
 	listenIPAddress   string
+	singleMachine     bool
 }
 
 type startArgs struct {
@@ -75,6 +79,7 @@ type startArgs struct {
 	outputDir     string
 	printMnemonic bool
 	rpcAddress    string
+	timeoutCommit time.Duration
 }
 
 func addTestnetFlagsToCmd(cmd *cobra.Command) {
@@ -147,6 +152,11 @@ Example:
 			args.listenIPAddress, _ = cmd.Flags().GetString(flagListenIPAddress)
 			args.numValidators, _ = cmd.Flags().GetInt(flagNumValidators)
 			args.algo, _ = cmd.Flags().GetString(flags.FlagKeyType)
+			args.singleMachine, _ = cmd.Flags().GetBool(flagSingleHost)
+			config.Consensus.TimeoutCommit, err = cmd.Flags().GetDuration(flagCommitTimeout)
+			if err != nil {
+				return err
+			}
 
 			return initTestnetFiles(clientCtx, cmd, config, mm, genBalIterator, clientCtx.TxConfig.SigningContext().ValidatorAddressCodec(), args)
 		},
@@ -158,6 +168,8 @@ Example:
 	cmd.Flags().String(flagStartingIPAddress, "192.168.0.1", "Starting IP address (192.168.0.1 results in persistent peers list ID0@192.168.0.1:46656, ID1@192.168.0.2:46656, ...)")
 	cmd.Flags().String(flagListenIPAddress, "127.0.0.1", "TCP or UNIX socket IP address for the RPC server to listen on")
 	cmd.Flags().String(flags.FlagKeyringBackend, flags.DefaultKeyringBackend, "Select keyring's backend (os|file|test)")
+	cmd.Flags().Duration(flagCommitTimeout, 5*time.Second, "Time to wait after a block commit before starting on the new height")
+	cmd.Flags().Bool(flagSingleHost, false, "Cluster runs on a single host machine with different ports")
 
 	return cmd
 }
@@ -231,26 +243,53 @@ func initTestnetFiles(
 		genBalances []banktypes.Balance
 		genFiles    []string
 	)
+	const (
+		rpcPort  = 26657
+		apiPort  = 1317
+		grpcPort = 9090
+	)
+	p2pPortStart := 26656
 
 	inBuf := bufio.NewReader(cmd.InOrStdin())
 	// generate private keys, node IDs, and initial transactions
 	for i := 0; i < args.numValidators; i++ {
+		var portOffset int
+		if args.singleMachine {
+			portOffset = i
+			p2pPortStart = 16656 // use different start point to not conflict with rpc port
+			nodeConfig.P2P.AddrBookStrict = false
+			nodeConfig.P2P.PexReactor = false
+			nodeConfig.P2P.AllowDuplicateIP = true
+		}
+
 		nodeDirName := fmt.Sprintf("%s%d", args.nodeDirPrefix, i)
 		nodeDir := filepath.Join(args.outputDir, nodeDirName, args.nodeDaemonHome)
 		gentxsDir := filepath.Join(args.outputDir, "gentxs")
 
 		nodeConfig.SetRoot(nodeDir)
 		nodeConfig.Moniker = nodeDirName
 		nodeConfig.RPC.ListenAddress = fmt.Sprintf("tcp://%s:26657", args.listenIPAddress)
+
+		simappConfig.API.Address = fmt.Sprintf("tcp://127.0.0.1:%d", apiPort+portOffset)
+		simappConfig.GRPC.Address = fmt.Sprintf("127.0.0.1:%d", grpcPort+portOffset)
+		simappConfig.GRPCWeb.Enable = true
+
 		if err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm); err != nil {
 			_ = os.RemoveAll(args.outputDir)
 			return err
 		}
-
-		ip, err := getIP(i, args.startingIPAddress)
-		if err != nil {
-			_ = os.RemoveAll(args.outputDir)
-			return err
+		var (
+			err error
+			ip  string
+		)
+		if args.singleMachine {
+			ip = "127.0.0.1"
+		} else {
+			ip, err = getIP(i, args.startingIPAddress)
+			if err != nil {
+				_ = os.RemoveAll(args.outputDir)
+				return err
+			}
 		}
 
 		nodeIDs[i], valPubKeys[i], err = genutil.InitializeNodeValidatorFiles(nodeConfig)
@@ -259,7 +298,7 @@ func initTestnetFiles(
 			return err
 		}
 
-		memo := fmt.Sprintf("%s@%s:26656", nodeIDs[i], ip)
+		memo := fmt.Sprintf("%s@%s:%d", nodeIDs[i], ip, p2pPortStart+portOffset)
 		genFiles = append(genFiles, nodeConfig.GenesisFile())
 
 		kb, err := keyring.New(sdk.KeyringServiceName(), args.keyringBackend, nodeDir, inBuf, clientCtx.Codec)
@@ -361,6 +400,7 @@ func initTestnetFiles(
 	err := collectGenFiles(
 		clientCtx, nodeConfig, args.chainID, nodeIDs, valPubKeys, args.numValidators,
 		args.outputDir, args.nodeDirPrefix, args.nodeDaemonHome, genBalIterator, valAddrCodec,
+		rpcPort, p2pPortStart, args.singleMachine,
 	)
 	if err != nil {
 		return err
@@ -418,15 +458,24 @@ func collectGenFiles(
 	clientCtx client.Context, nodeConfig *cmtconfig.Config, chainID string,
 	nodeIDs []string, valPubKeys []cryptotypes.PubKey, numValidators int,
 	outputDir, nodeDirPrefix, nodeDaemonHome string, genBalIterator banktypes.GenesisBalancesIterator, valAddrCodec address.ValidatorAddressCodec,
+	rpcPortStart, p2pPortStart int,
+	singleMachine bool,
 ) error {
 	var appState json.RawMessage
 	genTime := cmttime.Now()
 
 	for i := 0; i < numValidators; i++ {
+		var portOffset int
+		if singleMachine {
+			portOffset = i
+		}
+
 		nodeDirName := fmt.Sprintf("%s%d", nodeDirPrefix, i)
 		nodeDir := filepath.Join(outputDir, nodeDirName, nodeDaemonHome)
 		gentxsDir := filepath.Join(outputDir, "gentxs")
 		nodeConfig.Moniker = nodeDirName
+		nodeConfig.RPC.ListenAddress = fmt.Sprintf("tcp://127.0.0.1:%d", rpcPortStart+portOffset)
+		nodeConfig.P2P.ListenAddress = fmt.Sprintf("tcp://127.0.0.1:%d", p2pPortStart+portOffset)
 
 		nodeConfig.SetRoot(nodeDir)
 
@@ -515,6 +564,7 @@ func startTestnet(cmd *cobra.Command, args startArgs) error {
 	networkConfig.APIAddress = args.apiAddress
 	networkConfig.GRPCAddress = args.grpcAddress
 	networkConfig.PrintMnemonic = args.printMnemonic
+	networkConfig.TimeoutCommit = args.timeoutCommit
 	networkLogger := network.NewCLILogger(cmd)
 
 	baseDir := fmt.Sprintf("%s/%s", args.outputDir, networkConfig.ChainID)

simapp/upgrades.go

@@ -1,50 +1,52 @@
 package simapp
 
 import (
-	"context"
+	"fmt"
 
-	storetypes "cosmossdk.io/store/types"
-	"cosmossdk.io/x/accounts"
-	protocolpooltypes "cosmossdk.io/x/protocolpool/types"
-	upgradetypes "cosmossdk.io/x/upgrade/types"
+	"cosmossdk.io/simapp/upgrades"
+	"cosmossdk.io/simapp/upgrades/noop"
+	v051 "cosmossdk.io/simapp/upgrades/v051"
 
-	"github.com/cosmos/cosmos-sdk/types/module"
-	crisistypes "github.com/cosmos/cosmos-sdk/x/crisis/types"
+	upgradetypes "cosmossdk.io/x/upgrade/types"
 )
 
-// UpgradeName defines the on-chain upgrade name for the sample SimApp upgrade
-// from v0.50.x to v0.51.x
-//
-// NOTE: This upgrade defines a reference implementation of what an upgrade
-// could look like when an application is migrating from Cosmos SDK version
-// v0.50.x to v0.51.x.
-const UpgradeName = "v050-to-v051"
+// Upgrades list of chain upgrades
+var Upgrades = []upgrades.Upgrade[upgrades.AppKeepers]{v051.Upgrade}
 
+// RegisterUpgradeHandlers registers the chain upgrade handlers
 func (app SimApp) RegisterUpgradeHandlers() {
-	app.UpgradeKeeper.SetUpgradeHandler(
-		UpgradeName,
-		func(ctx context.Context, _ upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
-			return app.ModuleManager.RunMigrations(ctx, app.Configurator(), fromVM)
-		},
-	)
+	if len(Upgrades) == 0 {
+		// always have a unique upgrade registered for the current version to test in system tests or manual
+		Upgrades = append(Upgrades, noop.NewUpgrade[upgrades.AppKeepers](app.Version()))
+	}
 
+	var keepers upgrades.AppKeepers
+	app.GetStoreKeys()
+	// register all upgrade handlers
+	for _, upgrade := range Upgrades {
+		app.UpgradeKeeper.SetUpgradeHandler(
+			upgrade.UpgradeName,
+			upgrade.CreateUpgradeHandler(
+				app.ModuleManager,
+				app.Configurator(),
+				&keepers,
+			),
+		)
+	}
 	upgradeInfo, err := app.UpgradeKeeper.ReadUpgradeInfoFromDisk()
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("failed to read upgrade info from disk %s", err))
 	}
 
-	if upgradeInfo.Name == UpgradeName && !app.UpgradeKeeper.IsSkipHeight(upgradeInfo.Height) {
-		storeUpgrades := storetypes.StoreUpgrades{
-			Added: []string{
-				accounts.ModuleName,
-				protocolpooltypes.ModuleName,
-			},
-			Deleted: []string{
-				crisistypes.ModuleName, // The SDK discontinued the crisis module in v0.51.0
-			},
-		}
+	if app.UpgradeKeeper.IsSkipHeight(upgradeInfo.Height) {
+		return
+	}
 
-		// configure store loader that checks if version == upgradeHeight and applies store upgrades
-		app.SetStoreLoader(upgradetypes.UpgradeStoreLoader(upgradeInfo.Height, &storeUpgrades))
+	// register store loader for current upgrade
+	for _, upgrade := range Upgrades {
+		if upgradeInfo.Name == upgrade.UpgradeName {
+			app.SetStoreLoader(upgradetypes.UpgradeStoreLoader(upgradeInfo.Height, &upgrade.StoreUpgrades)) // nolint:gosec
+			break
+		}
 	}
 }

simapp/upgrades/noop/upgrades.go

@@ -0,0 +1,34 @@
+package noop
+
+import (
+	"context"
+
+	"cosmossdk.io/simapp/upgrades"
+
+	storetypes "cosmossdk.io/store/types"
+	upgradetypes "cosmossdk.io/x/upgrade/types"
+
+	"github.com/cosmos/cosmos-sdk/types/module"
+)
+
+// NewUpgrade constructor
+func NewUpgrade[T upgrades.AppKeepers](semver string) upgrades.Upgrade[T] {
+	return upgrades.Upgrade[T]{
+		UpgradeName:          semver,
+		CreateUpgradeHandler: CreateUpgradeHandler[T],
+		StoreUpgrades: storetypes.StoreUpgrades{
+			Added:   []string{},
+			Deleted: []string{},
+		},
+	}
+}
+
+func CreateUpgradeHandler[T upgrades.AppKeepers](
+	mm upgrades.ModuleManager,
+	configurator module.Configurator,
+	ak *T,
+) upgradetypes.UpgradeHandler {
+	return func(ctx context.Context, plan upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
+		return mm.RunMigrations(ctx, configurator, fromVM)
+	}
+}

simapp/upgrades/types.go

@@ -0,0 +1,31 @@
+package upgrades
+
+import (
+	"context"
+
+	storetypes "cosmossdk.io/store/types"
+	upgradetypes "cosmossdk.io/x/upgrade/types"
+
+	"github.com/cosmos/cosmos-sdk/types/module"
+)
+
+// AppKeepers placeholder type to customize for concrete use cases
+type AppKeepers any
+
+type ModuleManager interface {
+	RunMigrations(ctx context.Context, cfg module.Configurator, fromVM module.VersionMap) (module.VersionMap, error)
+	GetVersionMap() module.VersionMap
+}
+
+// Upgrade defines a struct containing necessary fields that a SoftwareUpgradeProposal
+// must have written, in order for the state migration to go smoothly.
+// An upgrade must implement this struct, and then set it in the app.go.
+// The app.go will then define the handler.
+type Upgrade[T AppKeepers] struct {
+	// Upgrade version name, for the upgrade handler, e.g. `v0.51.0`
+	UpgradeName string
+
+	// CreateUpgradeHandler defines the function that creates an upgrade handler
+	CreateUpgradeHandler func(ModuleManager, module.Configurator, *T) upgradetypes.UpgradeHandler
+	StoreUpgrades        storetypes.StoreUpgrades
+}

simapp/upgrades/v051/upgrades.go

@@ -0,0 +1,43 @@
+package v051
+
+import (
+	"context"
+
+	"cosmossdk.io/simapp/upgrades"
+	"cosmossdk.io/x/accounts"
+	protocolpooltypes "cosmossdk.io/x/protocolpool/types"
+	crisistypes "github.com/cosmos/cosmos-sdk/x/crisis/types"
+
+	storetypes "cosmossdk.io/store/types"
+	upgradetypes "cosmossdk.io/x/upgrade/types"
+
+	"github.com/cosmos/cosmos-sdk/types/module"
+)
+
+// UpgradeName defines the on-chain upgrade name
+const UpgradeName = "v0.51"
+
+var Upgrade = upgrades.Upgrade[upgrades.AppKeepers]{
+	UpgradeName:          UpgradeName,
+	CreateUpgradeHandler: CreateUpgradeHandler[upgrades.AppKeepers],
+	StoreUpgrades: storetypes.StoreUpgrades{
+		Added: []string{
+			accounts.StoreKey,
+			// accounts.ModuleName,
+			protocolpooltypes.ModuleName,
+		},
+		Deleted: []string{
+			crisistypes.ModuleName, // The SDK discontinued the crisis module in v0.51.0
+		},
+	},
+}
+
+func CreateUpgradeHandler[T upgrades.AppKeepers](
+	mm upgrades.ModuleManager,
+	configurator module.Configurator,
+	ak *T,
+) upgradetypes.UpgradeHandler {
+	return func(ctx context.Context, plan upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
+		return mm.RunMigrations(ctx, configurator, fromVM)
+	}
+}

systemtests/.gitignore

@@ -0,0 +1,2 @@
+/testnet
+/binaries

systemtests/Makefile

@@ -0,0 +1,15 @@
+#!/usr/bin/make -f
+
+WAIT_TIME ?= 45s
+
+all: test
+
+test:
+	go test -mod=readonly -failfast -tags='system_test' ./... --wait-time=$(WAIT_TIME) --verbose
+
+format:
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gofumpt -w
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs misspell -w
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "./tests/system/vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gci write --skip-generated -s standard -s default -s "prefix(cosmossdk.io)" -s "prefix(github.com/cosmos/cosmos-sdk)" --custom-order
+
+.PHONY: all test format