Backport of [QT-616] Add seal_ha enos scenario into release/1.14.x

.github/workflows/enos-lint.yml

@@ -41,7 +41,6 @@ jobs:
       - uses: hashicorp/setup-terraform@v2
         with:
           terraform_wrapper: false
-          terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved
       - uses: hashicorp/action-setup-enos@v1
         with:
           github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}

.github/workflows/enos-run-k8s.yml

@@ -38,7 +38,6 @@ jobs:
           # the Terraform wrapper will break Terraform execution in Enos because
           # it changes the output to text when we expect it to be JSON.
           terraform_wrapper: false
-          terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved
       - name: Set up Enos
         uses: hashicorp/action-setup-enos@v1
         with:

.github/workflows/test-enos-scenario-ui.yml

@@ -85,7 +85,6 @@ jobs:
         with:
           cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
           terraform_wrapper: false
-          terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved
       - name: Prepare scenario dependencies
         run: |
           mkdir -p ./enos/support/terraform-plugin-cache

.github/workflows/test-run-enos-scenario-matrix.yml

@@ -102,7 +102,6 @@ jobs:
           # the Terraform wrapper will break Terraform execution in Enos because
           # it changes the output to text when we expect it to be JSON.
           terraform_wrapper: false
-          terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved
       - uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_CI }}

enos/ci/service-user-iam/main.tf

@@ -216,6 +216,7 @@ data "aws_iam_policy_document" "enos_scenario" {
       "kms:ListKeys",
       "kms:ListResourceTags",
       "kms:ScheduleKeyDeletion",
+      "kms:TagResource",
       "servicequotas:ListServiceQuotas"
     ]
 

enos/enos-modules.hcl

@@ -57,6 +57,18 @@ module "replication_data" {
   source = "./modules/replication_data"
 }
 
+module "seal_key_awskms" {
+  source = "./modules/seal_key_awskms"
+
+  common_tags = var.tags
+}
+
+module "seal_key_shamir" {
+  source = "./modules/seal_key_shamir"
+
+  common_tags = var.tags
+}
+
 module "shutdown_node" {
   source = "./modules/shutdown_node"
 }
@@ -65,6 +77,17 @@ module "shutdown_multiple_nodes" {
   source = "./modules/shutdown_multiple_nodes"
 }
 
+module "start_vault" {
+  source = "./modules/start_vault"
+
+  install_dir = var.vault_install_dir
+  log_level   = var.vault_log_level
+}
+
+module "stop_vault" {
+  source = "./modules/stop_vault"
+}
+
 # create target instances using ec2:CreateFleet
 module "target_ec2_fleet" {
   source = "./modules/target_ec2_fleet"
@@ -249,3 +272,10 @@ module "vault_wait_for_leader" {
   vault_install_dir    = var.vault_install_dir
   vault_instance_count = var.vault_instance_count
 }
+
+module "vault_wait_for_seal_rewrap" {
+  source = "./modules/vault_wait_for_seal_rewrap"
+
+  vault_install_dir    = var.vault_install_dir
+  vault_instance_count = var.vault_instance_count
+}

enos/enos-scenario-agent.hcl

@@ -11,6 +11,7 @@ scenario "agent" {
     distro          = ["ubuntu", "rhel"]
     edition         = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
     seal            = ["awskms", "shamir"]
+    seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
     exclude {
@@ -81,6 +82,15 @@ scenario "agent" {
     }
   }
 
+  step "create_seal_key" {
+    module = "seal_key_${matrix.seal}"
+
+    variables {
+      cluster_id  = step.create_vpc.cluster_id
+      common_tags = global.tags
+    }
+  }
+
   // This step reads the contents of the backend license if we're using a Consul backend and
   // the edition is "ent".
   step "read_backend_license" {
@@ -110,11 +120,11 @@ scenario "agent" {
     }
 
     variables {
-      ami_id                = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
-      cluster_tag_key       = global.vault_tag_key
-      common_tags           = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      ami_id          = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
+      cluster_tag_key = global.vault_tag_key
+      common_tags     = global.tags
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -127,11 +137,11 @@ scenario "agent" {
     }
 
     variables {
-      ami_id                = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
-      cluster_tag_key       = global.backend_tag_key
-      common_tags           = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      ami_id          = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"]
+      cluster_tag_key = global.backend_tag_key
+      common_tags     = global.tags
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -171,7 +181,6 @@ scenario "agent" {
 
     variables {
       artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
-      awskms_unseal_key_arn   = step.create_vpc.kms_key_arn
       backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       cluster_name            = step.create_vault_cluster_targets.cluster_name
@@ -186,9 +195,11 @@ scenario "agent" {
       local_artifact_path  = local.artifact_path
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
+      seal_ha_beta         = matrix.seal_ha_beta
+      seal_key_name        = step.create_seal_key.resource_name
+      seal_type            = matrix.seal
       storage_backend      = matrix.backend
       target_hosts         = step.create_vault_cluster_targets.hosts
-      unseal_method        = matrix.seal
     }
   }
 
@@ -389,11 +400,6 @@ scenario "agent" {
     value       = step.create_vault_cluster.audit_device_file_path
   }
 
-  output "awskms_unseal_key_arn" {
-    description = "The Vault cluster KMS key arn"
-    value       = step.create_vpc.kms_key_arn
-  }
-
   output "cluster_name" {
     description = "The Vault cluster name"
     value       = step.create_vault_cluster.cluster_name
@@ -434,6 +440,11 @@ scenario "agent" {
     value       = step.create_vault_cluster.recovery_keys_hex
   }
 
+  output "seal_key_name" {
+    description = "The name of the cluster seal key"
+    value       = step.create_seal_key.resource_name
+  }
+
   output "unseal_keys_b64" {
     description = "The Vault cluster unseal keys"
     value       = step.create_vault_cluster.unseal_keys_b64

enos/enos-scenario-autopilot.hcl

@@ -12,6 +12,7 @@ scenario "autopilot" {
     // release branch's version.
     initial_version = ["1.11.12", "1.12.11", "1.13.6", "1.14.2"]
     seal            = ["awskms", "shamir"]
+    seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
     exclude {
@@ -77,6 +78,15 @@ scenario "autopilot" {
     }
   }
 
+  step "create_seal_key" {
+    module = "seal_key_${matrix.seal}"
+
+    variables {
+      cluster_id  = step.create_vpc.cluster_id
+      common_tags = global.tags
+    }
+  }
+
   step "read_license" {
     module = module.read_license
 
@@ -94,11 +104,11 @@ scenario "autopilot" {
     }
 
     variables {
-      ami_id                = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
-      cluster_tag_key       = global.vault_tag_key
-      common_tags           = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      ami_id          = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
+      cluster_tag_key = global.vault_tag_key
+      common_tags     = global.tags
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -114,22 +124,23 @@ scenario "autopilot" {
     }
 
     variables {
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
-      cluster_name          = step.create_vault_cluster_targets.cluster_name
-      install_dir           = local.vault_install_dir
-      license               = matrix.edition != "ce" ? step.read_license.license : null
-      packages              = concat(global.packages, global.distro_packages[matrix.distro])
+      cluster_name         = step.create_vault_cluster_targets.cluster_name
+      enable_audit_devices = var.vault_enable_audit_devices
+      install_dir          = local.vault_install_dir
+      license              = matrix.edition != "ce" ? step.read_license.license : null
+      packages             = concat(global.packages, global.distro_packages[matrix.distro])
       release = {
         edition = matrix.edition
         version = matrix.initial_version
       }
+      seal_ha_beta    = matrix.seal_ha_beta
+      seal_key_name   = step.create_seal_key.resource_name
+      seal_type       = matrix.seal
       storage_backend = "raft"
       storage_backend_addl_config = {
         autopilot_upgrade_version = matrix.initial_version
       }
-      target_hosts         = step.create_vault_cluster_targets.hosts
-      unseal_method        = matrix.seal
-      enable_audit_devices = var.vault_enable_audit_devices
+      target_hosts = step.create_vault_cluster_targets.hosts
     }
   }
 
@@ -190,11 +201,11 @@ scenario "autopilot" {
     }
 
     variables {
-      ami_id                = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
-      common_tags           = global.tags
-      cluster_name          = step.create_vault_cluster_targets.cluster_name
-      vpc_id                = step.create_vpc.vpc_id
+      ami_id         = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
+      common_tags    = global.tags
+      cluster_name   = step.create_vault_cluster_targets.cluster_name
+      seal_key_names = step.create_seal_key.resource_names
+      vpc_id         = step.create_vpc.id
     }
   }
 
@@ -213,7 +224,7 @@ scenario "autopilot" {
 
     variables {
       artifactory_release         = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
-      awskms_unseal_key_arn       = step.create_vpc.kms_key_arn
+      enable_audit_devices        = var.vault_enable_audit_devices
       cluster_name                = step.create_vault_cluster_targets.cluster_name
       log_level                   = var.vault_log_level
       force_unseal                = matrix.seal == "shamir"
@@ -224,13 +235,14 @@ scenario "autopilot" {
       manage_service              = local.manage_service
       packages                    = concat(global.packages, global.distro_packages[matrix.distro])
       root_token                  = step.create_vault_cluster.root_token
+      seal_ha_beta                = matrix.seal_ha_beta
+      seal_key_name               = step.create_seal_key.resource_name
+      seal_type                   = matrix.seal
       shamir_unseal_keys          = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null
       storage_backend             = "raft"
       storage_backend_addl_config = step.create_autopilot_upgrade_storageconfig.storage_addl_config
       storage_node_prefix         = "upgrade_node"
       target_hosts                = step.create_vault_cluster_upgrade_targets.hosts
-      unseal_method               = matrix.seal
-      enable_audit_devices        = var.vault_enable_audit_devices
     }
   }
 
@@ -498,9 +510,9 @@ scenario "autopilot" {
     }
   }
 
-  output "awskms_unseal_key_arn" {
-    description = "The Vault cluster KMS key arn"
-    value       = step.create_vpc.kms_key_arn
+  output "audit_device_file_path" {
+    description = "The file path for the file audit device, if enabled"
+    value       = step.create_vault_cluster.audit_device_file_path
   }
 
   output "cluster_name" {
@@ -543,6 +555,11 @@ scenario "autopilot" {
     value       = step.create_vault_cluster.recovery_keys_hex
   }
 
+  output "seal_key_name" {
+    description = "The Vault cluster seal key name"
+    value       = step.create_seal_key.resource_name
+  }
+
   output "unseal_keys_b64" {
     description = "The Vault cluster unseal keys"
     value       = step.create_vault_cluster.unseal_keys_b64
@@ -567,9 +584,4 @@ scenario "autopilot" {
     description = "The Vault cluster public IPs"
     value       = step.upgrade_vault_cluster_with_autopilot.public_ips
   }
-
-  output "vault_audit_device_file_path" {
-    description = "The file path for the file audit device, if enabled"
-    value       = step.create_vault_cluster.audit_device_file_path
-  }
 }