diff --git a/changelog/29228.txt b/changelog/29228.txt new file mode 100644 index 000000000000..d60e1349425a --- /dev/null +++ b/changelog/29228.txt @@ -0,0 +1,3 @@ +```release-note:change +server/config: Configuration values including IPv6 addresses will be automatically translated and displayed conformant to RFC-5952 §4. +``` diff --git a/command/operator_migrate_test.go b/command/operator_migrate_test.go index 15190b2640f5..48dedb6080c0 100644 --- a/command/operator_migrate_test.go +++ b/command/operator_migrate_test.go @@ -190,23 +190,23 @@ func TestMigration(t *testing.T) { cmd := new(OperatorMigrateCommand) cfgName := filepath.Join(t.TempDir(), "migrator") os.WriteFile(cfgName, []byte(` -storage_source "src_type" { +storage_source "consul" { path = "src_path" } -storage_destination "dest_type" { +storage_destination "raft" { path = "dest_path" }`), 0o644) expCfg := &migratorConfig{ StorageSource: &server.Storage{ - Type: "src_type", + Type: "consul", Config: map[string]string{ "path": "src_path", }, }, StorageDestination: &server.Storage{ - Type: "dest_type", + Type: "raft", Config: map[string]string{ "path": "dest_path", }, @@ -230,41 +230,41 @@ storage_destination "dest_type" { // missing source verifyBad(` -storage_destination "dest_type" { +storage_destination "raft" { path = "dest_path" }`) // missing destination verifyBad(` -storage_source "src_type" { +storage_source "consul" { path = "src_path" }`) // duplicate source verifyBad(` -storage_source "src_type" { +storage_source "consul" { path = "src_path" } -storage_source "src_type2" { +storage_source "raft" { path = "src_path" } -storage_destination "dest_type" { +storage_destination "raft" { path = "dest_path" }`) // duplicate destination verifyBad(` -storage_source "src_type" { +storage_source "consul" { path = "src_path" } -storage_destination "dest_type" { +storage_destination "raft" { path = "dest_path" } -storage_destination "dest_type2" { +storage_destination "consul" { path = "dest_path" }`) }) diff --git a/command/server.go b/command/server.go index c07b1acc959b..10bd6fc9272e 100644 --- a/command/server.go +++ b/command/server.go @@ -514,7 +514,7 @@ func (c *ServerCommand) runRecoveryMode() int { } if config.Storage.Type == storageTypeRaft || (config.HAStorage != nil && config.HAStorage.Type == storageTypeRaft) { if envCA := os.Getenv("VAULT_CLUSTER_ADDR"); envCA != "" { - config.ClusterAddr = envCA + config.ClusterAddr = configutil.NormalizeAddr(envCA) } if len(config.ClusterAddr) == 0 { @@ -742,9 +742,9 @@ func (c *ServerCommand) runRecoveryMode() int { func logProxyEnvironmentVariables(logger hclog.Logger) { proxyCfg := httpproxy.FromEnvironment() cfgMap := map[string]string{ - "http_proxy": proxyCfg.HTTPProxy, - "https_proxy": proxyCfg.HTTPSProxy, - "no_proxy": proxyCfg.NoProxy, + "http_proxy": configutil.NormalizeAddr(proxyCfg.HTTPProxy), + "https_proxy": configutil.NormalizeAddr(proxyCfg.HTTPSProxy), + "no_proxy": configutil.NormalizeAddr(proxyCfg.NoProxy), } for k, v := range cfgMap { u, err := url.Parse(v) @@ -2243,7 +2243,7 @@ func (c *ServerCommand) detectRedirect(detect physical.RedirectDetect, } // Return the URL string - return url.String(), nil + return configutil.NormalizeAddr(url.String()), nil } func (c *ServerCommand) Reload(lock *sync.RWMutex, reloadFuncs *map[string][]reloadutil.ReloadFunc, configPath []string, core *vault.Core) error { @@ -2749,11 +2749,11 @@ func initHaBackend(c *ServerCommand, config *server.Config, coreConfig *vault.Co func determineRedirectAddr(c *ServerCommand, coreConfig *vault.CoreConfig, config *server.Config) error { var retErr error if envRA := os.Getenv("VAULT_API_ADDR"); envRA != "" { - coreConfig.RedirectAddr = envRA + coreConfig.RedirectAddr = configutil.NormalizeAddr(envRA) } else if envRA := os.Getenv("VAULT_REDIRECT_ADDR"); envRA != "" { - coreConfig.RedirectAddr = envRA + coreConfig.RedirectAddr = configutil.NormalizeAddr(envRA) } else if envAA := os.Getenv("VAULT_ADVERTISE_ADDR"); envAA != "" { - coreConfig.RedirectAddr = envAA + coreConfig.RedirectAddr = configutil.NormalizeAddr(envAA) } // Attempt to detect the redirect address, if possible @@ -2785,7 +2785,7 @@ func determineRedirectAddr(c *ServerCommand, coreConfig *vault.CoreConfig, confi if c.flagDevTLS { protocol = "https" } - coreConfig.RedirectAddr = fmt.Sprintf("%s://%s", protocol, config.Listeners[0].Address) + coreConfig.RedirectAddr = configutil.NormalizeAddr(fmt.Sprintf("%s://%s", protocol, config.Listeners[0].Address)) } return retErr } @@ -2794,7 +2794,7 @@ func findClusterAddress(c *ServerCommand, coreConfig *vault.CoreConfig, config * if disableClustering { coreConfig.ClusterAddr = "" } else if envCA := os.Getenv("VAULT_CLUSTER_ADDR"); envCA != "" { - coreConfig.ClusterAddr = envCA + coreConfig.ClusterAddr = configutil.NormalizeAddr(envCA) } else { var addrToUse string switch { @@ -2826,7 +2826,7 @@ func findClusterAddress(c *ServerCommand, coreConfig *vault.CoreConfig, config * u.Host = net.JoinHostPort(host, strconv.Itoa(nPort+1)) // Will always be TLS-secured u.Scheme = "https" - coreConfig.ClusterAddr = u.String() + coreConfig.ClusterAddr = configutil.NormalizeAddr(u.String()) } CLUSTER_SYNTHESIS_COMPLETE: diff --git a/command/server/config.go b/command/server/config.go index 4c1951c0249a..f1c0de11cf82 100644 --- a/command/server/config.go +++ b/command/server/config.go @@ -11,6 +11,7 @@ import ( "math" "os" "path/filepath" + "slices" "strconv" "strings" "time" @@ -934,33 +935,49 @@ func ParseStorage(result *Config, list *ast.ObjectList, name string) error { } m := make(map[string]string) - for key, val := range config { - valStr, ok := val.(string) + for k, v := range config { + vStr, ok := v.(string) if ok { - m[key] = valStr + var err error + m[k], err = normalizeStorageConfigAddresses(key, k, vStr) + if err != nil { + return err + } continue } - valBytes, err := json.Marshal(val) - if err != nil { - return err + + var err error + var vBytes []byte + // Raft's retry_join requires special normalization due to its complexity + if key == "raft" && k == "retry_join" { + vBytes, err = normalizeRaftRetryJoin(v) + if err != nil { + return err + } + } else { + vBytes, err = json.Marshal(v) + if err != nil { + return err + } } - m[key] = string(valBytes) + + m[k] = string(vBytes) } // Pull out the redirect address since it's common to all backends var redirectAddr string if v, ok := m["redirect_addr"]; ok { - redirectAddr = v + redirectAddr = configutil.NormalizeAddr(v) delete(m, "redirect_addr") } else if v, ok := m["advertise_addr"]; ok { - redirectAddr = v + redirectAddr = configutil.NormalizeAddr(v) delete(m, "advertise_addr") } // Pull out the cluster address since it's common to all backends var clusterAddr string if v, ok := m["cluster_addr"]; ok { - clusterAddr = v + clusterAddr = configutil.NormalizeAddr(v) delete(m, "cluster_addr") } @@ -997,6 +1014,120 @@ func ParseStorage(result *Config, list *ast.ObjectList, name string) error { return nil } +// storageAddressKeys maps a storage backend type to its associated +// configuration whose values are URLs, IP addresses, or host:port style +// addresses. All physical storage types must have an entry in this map, +// otherwise our normalization check will fail when parsing the storage entry +// config. Physical storage types which don't contain such keys should include +// an empty array. +var storageAddressKeys = map[string][]string{ + "aerospike": {"hostname"}, + "alicloudoss": {"endpoint"}, + "azure": {"arm_endpoint"}, + "cassandra": {"hosts"}, + "cockroachdb": {"connection_url"}, + "consul": {"address", "service_address"}, + "couchdb": {"endpoint"}, + "dynamodb": {"endpoint"}, + "etcd": {"address", "discovery_srv"}, + "file": {}, + "filesystem": {}, + "foundationdb": {}, + "gcs": {}, + "inmem": {}, + "inmem_ha": {}, + "inmem_transactional": {}, + "inmem_transactional_ha": {}, + "manta": {"url"}, + "mssql": {"server"}, + "mysql": {"address"}, + "oci": {}, + "postgresql": {"connection_url"}, + "raft": {}, // retry_join is handled separately in normalizeRaftRetryJoin() + "s3": {"endpoint"}, + "spanner": {}, + "swift": {"auth_url", "storage_url"}, + "zookeeper": {"address"}, +} + +// normalizeStorageConfigAddresses takes a storage name, a configuration key +// and it's associated value and will normalize any URLs, IP addresses, or +// host:port style addresses. +func normalizeStorageConfigAddresses(storage string, key string, value string) (string, error) { + keys, ok := storageAddressKeys[storage] + if !ok { + return "", fmt.Errorf("unknown storage type %s", storage) + } + + if slices.Contains(keys, key) { + return configutil.NormalizeAddr(value), nil + } + + return value, nil +} + +// normalizeRaftRetryJoin takes the hcl decoded value representation of a +// retry_join stanza and normalizes any URLs, IP addresses, or host:port style +// addresses, and returns the value encoded as JSON. +func normalizeRaftRetryJoin(val any) ([]byte, error) { + res := []map[string]any{} + + // Depending on whether the retry_join stanzas were configured as an attribute, + // a block, or a mixture of both, we'll get different values from which we + // need to extract our individual retry joins stanzas. + stanzas := []map[string]any{} + if retryJoin, ok := val.([]map[string]any); ok { + // retry_join stanzas are defined as blocks + stanzas = retryJoin + } else { + // retry_join stanzas are defined as attributes or attributes and blocks + retryJoin, ok := val.([]any) + if !ok { + // retry_join stanzas have not been configured correctly + return nil, fmt.Errorf("malformed retry_join stanza: %v", val) + } + + for _, stanza := range retryJoin { + stanzaVal, ok := stanza.(map[string]any) + if !ok { + return nil, fmt.Errorf("malformed retry_join stanza: %v", stanza) + } + stanzas = append(stanzas, stanzaVal) + } + } + + for _, stanza := range stanzas { + normalizedStanza := map[string]any{} + for k, v := range stanza { + switch k { + case "auto_join": + pairs := strings.Split(v.(string), " ") + for i, pair := range pairs { + pairParts := strings.Split(pair, "=") + if len(pairParts) != 2 { + return nil, fmt.Errorf("malformed auto_join pair %s, expected key=value", pair) + } + // These are auto_join keys that are valid for the provider in go-discover + if slices.Contains([]string{"domain", "auth_url", "url", "host"}, pairParts[0]) { + pairParts[1] = configutil.NormalizeAddr(pairParts[1]) + pair = strings.Join(pairParts, "=") + pairs[i] = pair + } + } + normalizedStanza[k] = strings.Join(pairs, " ") + case "leader_api_addr": + normalizedStanza[k] = configutil.NormalizeAddr(v.(string)) + default: + normalizedStanza[k] = v + } + } + + res = append(res, normalizedStanza) + } + + return json.Marshal(res) +} + func parseHAStorage(result *Config, list *ast.ObjectList, name string) error { if len(list.Items) > 1 { return fmt.Errorf("only one %q block is permitted", name) @@ -1016,33 +1147,49 @@ func parseHAStorage(result *Config, list *ast.ObjectList, name string) error { } m := make(map[string]string) - for key, val := range config { - valStr, ok := val.(string) + for k, v := range config { + vStr, ok := v.(string) if ok { - m[key] = valStr + var err error + m[k], err = normalizeStorageConfigAddresses(key, k, vStr) + if err != nil { + return err + } continue } - valBytes, err := json.Marshal(val) - if err != nil { - return err + + var err error + var vBytes []byte + // Raft's retry_join requires special normalization due to its complexity + if key == "raft" && k == "retry_join" { + vBytes, err = normalizeRaftRetryJoin(v) + if err != nil { + return err + } + } else { + vBytes, err = json.Marshal(v) + if err != nil { + return err + } } - m[key] = string(valBytes) + + m[k] = string(vBytes) } // Pull out the redirect address since it's common to all backends var redirectAddr string if v, ok := m["redirect_addr"]; ok { - redirectAddr = v + redirectAddr = configutil.NormalizeAddr(v) delete(m, "redirect_addr") } else if v, ok := m["advertise_addr"]; ok { - redirectAddr = v + redirectAddr = configutil.NormalizeAddr(v) delete(m, "advertise_addr") } // Pull out the cluster address since it's common to all backends var clusterAddr string if v, ok := m["cluster_addr"]; ok { - clusterAddr = v + clusterAddr = configutil.NormalizeAddr(v) delete(m, "cluster_addr") } @@ -1096,6 +1243,12 @@ func parseServiceRegistration(result *Config, list *ast.ObjectList, name string) return multierror.Prefix(err, fmt.Sprintf("%s.%s:", name, key)) } + if key == "consul" { + if addr, ok := m["address"]; ok { + m["address"] = configutil.NormalizeAddr(addr) + } + } + result.ServiceRegistration = &ServiceRegistration{ Type: strings.ToLower(key), Config: m, diff --git a/command/server/config_test.go b/command/server/config_test.go index 9fa20b182fd2..ab0554697d79 100644 --- a/command/server/config_test.go +++ b/command/server/config_test.go @@ -65,6 +65,14 @@ func TestParseStorage(t *testing.T) { testParseStorageTemplate(t) } +// TestParseStorageURLConformance tests that all config attrs whose values can be +// URLs, IP addresses, or host:port addresses, when configured with an IPv6 +// address, the normalized to be conformant with RFC-5942 §4 +// See: https://rfc-editor.org/rfc/rfc5952.html +func TestParseStorageURLConformance(t *testing.T) { + testParseStorageURLConformance(t) +} + // TestConfigWithAdministrativeNamespace tests that .hcl and .json configurations are correctly parsed when the administrative_namespace_path is present. func TestConfigWithAdministrativeNamespace(t *testing.T) { testConfigWithAdministrativeNamespaceHcl(t) diff --git a/command/server/config_test_helpers.go b/command/server/config_test_helpers.go index 258801dbfe4e..0153b888daaf 100644 --- a/command/server/config_test_helpers.go +++ b/command/server/config_test_helpers.go @@ -4,6 +4,7 @@ package server import ( + "encoding/json" "fmt" "reflect" "sort" @@ -29,36 +30,53 @@ func boolPointer(x bool) *bool { return &x } +// testConfigRaftRetryJoin decodes and normalizes retry_join stanzas. func testConfigRaftRetryJoin(t *testing.T) { - config, err := LoadConfigFile("./test-fixtures/raft_retry_join.hcl") - if err != nil { - t.Fatal(err) + t.Parallel() + + retryJoinExpected := []map[string]string{ + {"leader_api_addr": "http://127.0.0.1:8200"}, + {"leader_api_addr": "http://[2001:db8::2:1]:8200"}, + {"auto_join": "provider=mdns service=consul domain=2001:db8::2:1"}, + {"auto_join": "provider=os tag_key=consul tag_value=server username=foo password=bar auth_url=https://[2001:db8::2:1]/auth"}, + {"auto_join": "provider=triton account=testaccount url=https://[2001:db8::2:1] key_id=1234 tag_key=consul-role tag_value=server"}, + {"auto_join": "provider=packet auth_token=token project=uuid url=https://[2001:db8::2:1] address_type=public_v6"}, + {"auto_join": "provider=vsphere category_name=consul-role tag_name=consul-server host=https://[2001:db8::2:1] user=foo password=bar insecure_ssl=false"}, } - retryJoinConfig := `[{"leader_api_addr":"http://127.0.0.1:8200"},{"leader_api_addr":"http://127.0.0.2:8200"},{"leader_api_addr":"http://127.0.0.3:8200"}]` - expected := &Config{ - SharedConfig: &configutil.SharedConfig{ - Listeners: []*configutil.Listener{ + for _, cfg := range []string{ + "attr", + "block", + "mixed", + } { + t.Run(cfg, func(t *testing.T) { + t.Parallel() + + config, err := LoadConfigFile(fmt.Sprintf("./test-fixtures/raft_retry_join_%s.hcl", cfg)) + require.NoError(t, err) + retryJoinJSON, err := json.Marshal(retryJoinExpected) + require.NoError(t, err) + + expected := NewConfig() + expected.SharedConfig.Listeners = []*configutil.Listener{ { Type: "tcp", Address: "127.0.0.1:8200", CustomResponseHeaders: DefaultCustomHeaders, }, - }, - DisableMlock: true, - }, - - Storage: &Storage{ - Type: "raft", - Config: map[string]string{ - "path": "/storage/path/raft", - "node_id": "raft1", - "retry_join": retryJoinConfig, - }, - }, - } - config.Prune() - if diff := deep.Equal(config, expected); diff != nil { - t.Fatal(diff) + } + expected.SharedConfig.DisableMlock = true + expected.Storage = &Storage{ + Type: "raft", + Config: map[string]string{ + "path": "/storage/path/raft", + "node_id": "raft1", + "retry_join": string(retryJoinJSON), + }, + } + config.Prune() + require.EqualValues(t, expected.SharedConfig, config.SharedConfig) + require.EqualValues(t, expected.Storage, config.Storage) + }) } } @@ -143,7 +161,8 @@ func testLoadConfigFile_topLevel(t *testing.T, entropy *configutil.Entropy) { ServiceRegistration: &ServiceRegistration{ Type: "consul", Config: map[string]string{ - "foo": "bar", + "foo": "bar", + "address": "https://[2001:db8::1]:8500", }, }, @@ -1124,6 +1143,313 @@ ha_storage "consul" { } } +// testParseStorageURLConformance verifies that any storage configuration that +// takes a URL, IP Address, or host:port address conforms to RFC-5942 §4 when +// configured with an IPv6 address. See: https://rfc-editor.org/rfc/rfc5952.html +func testParseStorageURLConformance(t *testing.T) { + t.Parallel() + + for name, tc := range map[string]struct { + config string + expected *Storage + shouldFail bool + }{ + "aerospike": { + config: ` +storage "aerospike" { + hostname = "2001:db8:0:0:0:0:2:1" + port = "3000" + namespace = "test" + set = "vault" + username = "admin" + password = "admin" +}`, + expected: &Storage{ + Type: "aerospike", + Config: map[string]string{ + "hostname": "2001:db8::2:1", + "port": "3000", + "namespace": "test", + "set": "vault", + "username": "admin", + "password": "admin", + }, + }, + }, + "alicloudoss": { + config: ` +storage "alicloudoss" { + access_key = "abcd1234" + secret_key = "defg5678" + endpoint = "2001:db8:0:0:0:0:2:1" + bucket = "my-bucket" +}`, + expected: &Storage{ + Type: "alicloudoss", + Config: map[string]string{ + "access_key": "abcd1234", + "secret_key": "defg5678", + "endpoint": "2001:db8::2:1", + "bucket": "my-bucket", + }, + }, + }, + "azure": { + config: ` +storage "azure" { + accountName = "my-storage-account" + accountKey = "abcd1234" + arm_endpoint = "2001:db8:0:0:0:0:2:1" + container = "container-efgh5678" + environment = "AzurePublicCloud" +}`, + expected: &Storage{ + Type: "azure", + Config: map[string]string{ + "accountName": "my-storage-account", + "accountKey": "abcd1234", + "arm_endpoint": "2001:db8::2:1", + "container": "container-efgh5678", + "environment": "AzurePublicCloud", + }, + }, + }, + "cassandra": { + config: ` +storage "cassandra" { + hosts = "2001:db8:0:0:0:0:2:1" + consistency = "LOCAL_QUORUM" + protocol_version = 3 +}`, + expected: &Storage{ + Type: "cassandra", + Config: map[string]string{ + "hosts": "2001:db8::2:1", + "consistency": "LOCAL_QUORUM", + "protocol_version": "3", + }, + }, + }, + "cockroachdb": { + config: ` +storage "cockroachdb" { + connection_url = "postgres://user123:secret123!@2001:db8:0:0:0:0:2:1:5432/vault" + table = "vault_kv_store" +}`, + expected: &Storage{ + Type: "cockroachdb", + Config: map[string]string{ + "connection_url": "postgres://user123:secret123%21@[2001:db8::2:1]:5432/vault", + "table": "vault_kv_store", + }, + }, + }, + "consul": { + config: ` +storage "consul" { + address = "2001:db8:0:0:0:0:2:1:8500" + path = "vault/" +}`, + expected: &Storage{ + Type: "consul", + Config: map[string]string{ + "address": "2001:db8::2:1:8500", + "path": "vault/", + }, + }, + }, + "couchdb": { + config: ` +storage "couchdb" { + endpoint = "https://[2001:db8:0:0:0:0:2:1]:5984/my-database" + username = "admin" + password = "admin" +}`, + expected: &Storage{ + Type: "couchdb", + Config: map[string]string{ + "endpoint": "https://[2001:db8::2:1]:5984/my-database", + "username": "admin", + "password": "admin", + }, + }, + }, + "dynamodb": { + config: ` +storage "dynamodb" { + endpoint = "https://[2001:db8:0:0:0:0:2:1]:5984/my-aws-endpoint" + ha_enabled = "true" + region = "us-west-2" + table = "vault-data" +}`, + expected: &Storage{ + Type: "dynamodb", + Config: map[string]string{ + "endpoint": "https://[2001:db8::2:1]:5984/my-aws-endpoint", + "ha_enabled": "true", + "region": "us-west-2", + "table": "vault-data", + }, + }, + }, + "etcd": { + config: ` +storage "etcd" { + address = "https://[2001:db8:0:0:0:0:2:1]:2379" + discovery_srv = "https://[2001:db8:0:0:1:0:0:1]" + etcd_api = "v3" +}`, + expected: &Storage{ + Type: "etcd", + Config: map[string]string{ + "address": "https://[2001:db8::2:1]:2379", + "discovery_srv": "https://[2001:db8::1:0:0:1]", + "etcd_api": "v3", + }, + }, + }, + "manta": { + config: ` +storage "manta" { + directory = "manta-directory" + user = "myuser" + key_id = "40:9d:d3:f9:0b:86:62:48:f4:2e:a5:8e:43:00:2a:9b" + url = "https://[2001:db8:0:0:0:0:2:1]" +}`, + expected: &Storage{ + Type: "manta", + Config: map[string]string{ + "directory": "manta-directory", + "user": "myuser", + "key_id": "40:9d:d3:f9:0b:86:62:48:f4:2e:a5:8e:43:00:2a:9b", + "url": "https://[2001:db8::2:1]", + }, + }, + }, + "mssql": { + config: ` +storage "mssql" { + server = "2001:db8:0:0:0:0:2:1" + port = 1433 + username = "user1234" + password = "secret123!" + database = "vault" + table = "vault" + appname = "vault" + schema = "dbo" + connectionTimeout = 30 + logLevel = 0 +}`, + expected: &Storage{ + Type: "mssql", + Config: map[string]string{ + "server": "2001:db8::2:1", + "port": "1433", + "username": "user1234", + "password": "secret123!", + "database": "vault", + "table": "vault", + "appname": "vault", + "schema": "dbo", + "connectionTimeout": "30", + "logLevel": "0", + }, + }, + }, + "mysql": { + config: ` +storage "mysql" { + address = "2001:db8:0:0:0:0:2:1:3306" + username = "user1234" + password = "secret123!" + database = "vault" +}`, + expected: &Storage{ + Type: "mysql", + Config: map[string]string{ + "address": "2001:db8::2:1:3306", + "username": "user1234", + "password": "secret123!", + "database": "vault", + }, + }, + }, + "postgresql": { + config: ` +storage "postgresql" { + connection_url = "postgres://user123:secret123!@2001:db8:0:0:0:0:2:1:5432/vault" + table = "vault_kv_store" +}`, + expected: &Storage{ + Type: "postgresql", + Config: map[string]string{ + "connection_url": "postgres://user123:secret123%21@[2001:db8::2:1]:5432/vault", + "table": "vault_kv_store", + }, + }, + }, + "s3": { + config: ` +storage "s3" { + endpoint = "https://[2001:db8:0:0:0:0:2:1]:5984/my-aws-endpoint" + access_key = "abcd1234" + secret_key = "defg5678" + bucket = "my-bucket" +}`, + expected: &Storage{ + Type: "s3", + Config: map[string]string{ + "endpoint": "https://[2001:db8::2:1]:5984/my-aws-endpoint", + "access_key": "abcd1234", + "secret_key": "defg5678", + "bucket": "my-bucket", + }, + }, + }, + "swift": { + config: ` +storage "swift" { + auth_url = "https://[2001:db8:0:0:0:0:2:1]/auth" + storage_url = "https://[2001:db8:0:0:0:0:2:1]/storage" + username = "admin" + password = "secret123!" + container = "my-storage-container" +}`, + expected: &Storage{ + Type: "swift", + Config: map[string]string{ + "auth_url": "https://[2001:db8::2:1]/auth", + "storage_url": "https://[2001:db8::2:1]/storage", + "username": "admin", + "password": "secret123!", + "container": "my-storage-container", + }, + }, + }, + "zookeeper": { + config: ` +storage "zookeeper" { + address = "2001:db8:0:0:0:0:2:1:2181" + path = "vault/" +}`, + expected: &Storage{ + Type: "zookeeper", + Config: map[string]string{ + "address": "2001:db8::2:1:2181", + "path": "vault/", + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + config, err := ParseConfig(tc.config, "") + require.NoError(t, err) + require.EqualValues(t, tc.expected, config.Storage) + }) + } +} + func testParseSeals(t *testing.T) { config, err := LoadConfigFile("./test-fixtures/config_seals.hcl") if err != nil { diff --git a/command/server/test-fixtures/config2.hcl b/command/server/test-fixtures/config2.hcl index 0e383fb25910..7bf6c72a41c0 100644 --- a/command/server/test-fixtures/config2.hcl +++ b/command/server/test-fixtures/config2.hcl @@ -6,60 +6,61 @@ disable_mlock = true ui = true -api_addr = "top_level_api_addr" +api_addr = "top_level_api_addr" cluster_addr = "top_level_cluster_addr" listener "tcp" { - address = "127.0.0.1:443" + address = "127.0.0.1:443" } storage "consul" { - foo = "bar" - redirect_addr = "foo" + foo = "bar" + redirect_addr = "foo" } ha_storage "consul" { - bar = "baz" - redirect_addr = "snafu" - disable_clustering = "true" + bar = "baz" + redirect_addr = "snafu" + disable_clustering = "true" } service_registration "consul" { - foo = "bar" + foo = "bar" + address = "https://[2001:0db8::0001]:8500" } telemetry { - statsd_address = "bar" - usage_gauge_period = "5m" - maximum_gauge_cardinality = 125 - statsite_address = "foo" - dogstatsd_addr = "127.0.0.1:7254" - dogstatsd_tags = ["tag_1:val_1", "tag_2:val_2"] - prometheus_retention_time = "30s" + statsd_address = "bar" + usage_gauge_period = "5m" + maximum_gauge_cardinality = 125 + statsite_address = "foo" + dogstatsd_addr = "127.0.0.1:7254" + dogstatsd_tags = ["tag_1:val_1", "tag_2:val_2"] + prometheus_retention_time = "30s" } entropy "seal" { - mode = "augmentation" + mode = "augmentation" } sentinel { - additional_enabled_modules = [] + additional_enabled_modules = [] } kms "commastringpurpose" { - purpose = "foo,bar" + purpose = "foo,bar" } kms "slicepurpose" { - purpose = ["zip", "zap"] + purpose = ["zip", "zap"] } seal "nopurpose" { } seal "stringpurpose" { - purpose = "foo" + purpose = "foo" } -max_lease_ttl = "10h" -default_lease_ttl = "10h" -cluster_name = "testcluster" -pid_file = "./pidfile" +max_lease_ttl = "10h" +default_lease_ttl = "10h" +cluster_name = "testcluster" +pid_file = "./pidfile" raw_storage_endpoint = true -disable_sealwrap = true +disable_sealwrap = true diff --git a/command/server/test-fixtures/raft_retry_join.hcl b/command/server/test-fixtures/raft_retry_join.hcl deleted file mode 100644 index 844dd744e40c..000000000000 --- a/command/server/test-fixtures/raft_retry_join.hcl +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -storage "raft" { - path = "/storage/path/raft" - node_id = "raft1" - retry_join = [ - { - "leader_api_addr" = "http://127.0.0.1:8200" - }, - { - "leader_api_addr" = "http://127.0.0.2:8200" - }, - { - "leader_api_addr" = "http://127.0.0.3:8200" - } - ] -} -listener "tcp" { - address = "127.0.0.1:8200" -} -disable_mlock = true diff --git a/command/server/test-fixtures/raft_retry_join_attr.hcl b/command/server/test-fixtures/raft_retry_join_attr.hcl new file mode 100644 index 000000000000..883ba96b67d6 --- /dev/null +++ b/command/server/test-fixtures/raft_retry_join_attr.hcl @@ -0,0 +1,32 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +storage "raft" { + path = "/storage/path/raft" + node_id = "raft1" + retry_join = [ + { "leader_api_addr" = "http://127.0.0.1:8200" }, + { "leader_api_addr" = "http://[2001:db8:0:0:0:0:2:1]:8200" } + ] + retry_join = [ + { "auto_join" = "provider=mdns service=consul domain=2001:db8:0:0:0:0:2:1" } + ] + retry_join = [ + { "auto_join" = "provider=os tag_key=consul tag_value=server username=foo password=bar auth_url=https://[2001:db8:0:0:0:0:2:1]/auth" } + ] + retry_join = [ + { "auto_join" = "provider=triton account=testaccount url=https://[2001:db8:0:0:0:0:2:1] key_id=1234 tag_key=consul-role tag_value=server" } + ] + retry_join = [ + { "auto_join" = "provider=packet auth_token=token project=uuid url=https://[2001:db8:0:0:0:0:2:1] address_type=public_v6" } + ] + retry_join = [ + { "auto_join" = "provider=vsphere category_name=consul-role tag_name=consul-server host=https://[2001:db8:0:0:0:0:2:1] user=foo password=bar insecure_ssl=false" } + ] +} + +listener "tcp" { + address = "127.0.0.1:8200" +} + +disable_mlock = true diff --git a/command/server/test-fixtures/raft_retry_join_block.hcl b/command/server/test-fixtures/raft_retry_join_block.hcl new file mode 100644 index 000000000000..762bd5fd90ec --- /dev/null +++ b/command/server/test-fixtures/raft_retry_join_block.hcl @@ -0,0 +1,35 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +storage "raft" { + path = "/storage/path/raft" + node_id = "raft1" + + retry_join { + "leader_api_addr" = "http://127.0.0.1:8200" + } + retry_join { + "leader_api_addr" = "http://[2001:db8:0:0:0:0:2:1]:8200" + } + retry_join { + "auto_join" = "provider=mdns service=consul domain=2001:db8:0:0:0:0:2:1" + } + retry_join { + "auto_join" = "provider=os tag_key=consul tag_value=server username=foo password=bar auth_url=https://[2001:db8:0:0:0:0:2:1]/auth" + } + retry_join { + "auto_join" = "provider=triton account=testaccount url=https://[2001:db8:0:0:0:0:2:1] key_id=1234 tag_key=consul-role tag_value=server" + } + retry_join { + "auto_join" = "provider=packet auth_token=token project=uuid url=https://[2001:db8:0:0:0:0:2:1] address_type=public_v6" + } + retry_join { + "auto_join" = "provider=vsphere category_name=consul-role tag_name=consul-server host=https://[2001:db8:0:0:0:0:2:1] user=foo password=bar insecure_ssl=false" + } +} + +listener "tcp" { + address = "127.0.0.1:8200" +} + +disable_mlock = true diff --git a/command/server/test-fixtures/raft_retry_join_mixed.hcl b/command/server/test-fixtures/raft_retry_join_mixed.hcl new file mode 100644 index 000000000000..9a5905d49820 --- /dev/null +++ b/command/server/test-fixtures/raft_retry_join_mixed.hcl @@ -0,0 +1,32 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +storage "raft" { + path = "/storage/path/raft" + node_id = "raft1" + retry_join = [ + { "leader_api_addr" = "http://127.0.0.1:8200" }, + { "leader_api_addr" = "http://[2001:db8:0:0:0:0:2:1]:8200" } + ] + retry_join { + "auto_join" = "provider=mdns service=consul domain=2001:db8:0:0:0:0:2:1" + } + retry_join = [ + { "auto_join" = "provider=os tag_key=consul tag_value=server username=foo password=bar auth_url=https://[2001:db8:0:0:0:0:2:1]/auth" } + ] + retry_join { + "auto_join" = "provider=triton account=testaccount url=https://[2001:db8:0:0:0:0:2:1] key_id=1234 tag_key=consul-role tag_value=server" + } + retry_join = [ + { "auto_join" = "provider=packet auth_token=token project=uuid url=https://[2001:db8:0:0:0:0:2:1] address_type=public_v6" } + ] + retry_join { + "auto_join" = "provider=vsphere category_name=consul-role tag_name=consul-server host=https://[2001:db8:0:0:0:0:2:1] user=foo password=bar insecure_ssl=false" + } +} + +listener "tcp" { + address = "127.0.0.1:8200" +} + +disable_mlock = true diff --git a/internalshared/configutil/kms.go b/internalshared/configutil/kms.go index f0948118dd95..9a18616beec0 100644 --- a/internalshared/configutil/kms.go +++ b/internalshared/configutil/kms.go @@ -11,6 +11,7 @@ import ( "io" "os" "regexp" + "slices" "strings" "github.com/hashicorp/errwrap" @@ -157,7 +158,10 @@ func parseKMS(result *[]*KMS, list *ast.ObjectList, blockName string, maxKMS int if err != nil { return multierror.Prefix(err, fmt.Sprintf("%s.%s:", blockName, key)) } - strMap[k] = s + strMap[k], err = normalizeKMSSealConfigAddrs(key, k, s) + if err != nil { + return multierror.Prefix(err, fmt.Sprintf("%s.%s:", blockName, key)) + } } seal := &KMS{ @@ -214,27 +218,76 @@ func ParseKMSes(d string) ([]*KMS, error) { return result.Seals, nil } -func configureWrapper(configKMS *KMS, infoKeys *[]string, info *map[string]string, logger hclog.Logger, opts ...wrapping.Option) (wrapping.Wrapper, error) { - var wrapper wrapping.Wrapper - var kmsInfo map[string]string - var err error +// kmsSealAddressKeys maps seal key types to corresponding config keys whose +// values might contain URLs, IP addresses, or host:port addresses. All seal +// types must contain an entry here, otherwise our normalization check will fail +// when parsing the seal config. Seal types which do not contain such +// configurations ought to have an empty array as the value in the map. +var kmsSealAddressKeys = map[string][]string{ + wrapping.WrapperTypeAliCloudKms.String(): {"domain"}, + wrapping.WrapperTypeAwsKms.String(): {"endpoint"}, + wrapping.WrapperTypeAzureKeyVault.String(): {"resource"}, + wrapping.WrapperTypeGcpCkms.String(): {}, + wrapping.WrapperTypeOciKms.String(): {"key_id", "crypto_endpoint", "management_endpoint"}, + wrapping.WrapperTypePkcs11.String(): {}, + wrapping.WrapperTypeTransit.String(): {"address"}, +} +// normalizeKMSSealConfigAddrs takes a kms seal type, a config key, and its +// associated value and will normalize any URLs, IP addresses, or host:port +// addresses contained in the value if the config key is known in the +// kmsSealAddressKeys. +func normalizeKMSSealConfigAddrs(seal string, key string, value string) (string, error) { + keys, ok := kmsSealAddressKeys[seal] + if !ok { + return "", fmt.Errorf("unknown seal type %s", seal) + } + + if slices.Contains(keys, key) { + return NormalizeAddr(value), nil + } + + return value, nil +} + +// mergeKMSEnvConfig takes a KMS and merges any normalized values set via +// environment variables. +func mergeKMSEnvConfig(configKMS *KMS) error { envConfig := GetEnvConfigFunc(configKMS) if len(envConfig) > 0 && configKMS.Config == nil { configKMS.Config = make(map[string]string) } // transit is a special case, because some config values take precedence over env vars if configKMS.Type == wrapping.WrapperTypeTransit.String() { - mergeTransitConfig(configKMS.Config, envConfig) + if err := mergeTransitConfig(configKMS.Config, envConfig); err != nil { + return err + } } else { for name, val := range envConfig { - configKMS.Config[name] = val + var err error + configKMS.Config[name], err = normalizeKMSSealConfigAddrs(configKMS.Type, name, val) + if err != nil { + return err + } } } + return nil +} + +func configureWrapper(configKMS *KMS, infoKeys *[]string, info *map[string]string, logger hclog.Logger, opts ...wrapping.Option) (wrapping.Wrapper, error) { + var wrapper wrapping.Wrapper + var kmsInfo map[string]string + var err error + + // Get any seal config set as env variables and merge it into the KMS. + if err = mergeKMSEnvConfig(configKMS); err != nil { + return nil, err + } + switch wrapping.WrapperType(configKMS.Type) { case wrapping.WrapperTypeShamir: - return nil, nil + return wrapper, nil case wrapping.WrapperTypeAead: wrapper, kmsInfo, err = GetAEADKMSFunc(configKMS, opts...) @@ -456,7 +509,7 @@ func getEnvConfig(kms *KMS) map[string]string { return envValues } -func mergeTransitConfig(config map[string]string, envConfig map[string]string) { +func mergeTransitConfig(config map[string]string, envConfig map[string]string) error { useFileTlsConfig := false for _, varName := range TransitTLSConfigVars { if _, ok := config[varName]; ok { @@ -471,14 +524,20 @@ func mergeTransitConfig(config map[string]string, envConfig map[string]string) { } } + var err error for varName, val := range envConfig { // for some values, file config takes precedence if strutil.StrListContains(TransitPrioritizeConfigValues, varName) && config[varName] != "" { continue } - config[varName] = val + config[varName], err = normalizeKMSSealConfigAddrs(wrapping.WrapperTypeTransit.String(), varName, val) + if err != nil { + return err + } } + + return nil } func (k *KMS) Clone() *KMS { diff --git a/internalshared/configutil/kms_test.go b/internalshared/configutil/kms_test.go index 9eb19a3e3d7d..91c7ca3454da 100644 --- a/internalshared/configutil/kms_test.go +++ b/internalshared/configutil/kms_test.go @@ -4,9 +4,11 @@ package configutil import ( - "os" "reflect" "testing" + + "github.com/hashicorp/go-kms-wrapping/wrappers/ocikms/v2" + "github.com/stretchr/testify/require" ) func Test_getEnvConfig(t *testing.T) { @@ -83,20 +85,377 @@ func Test_getEnvConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { for envName, envVal := range tt.envVars { - if err := os.Setenv(envName, envVal); err != nil { - t.Errorf("error setting environment vars for test: %s", err) - } + t.Setenv(envName, envVal) } if got := GetEnvConfigFunc(tt.kms); !reflect.DeepEqual(got, tt.want) { t.Errorf("getEnvConfig() = %v, want %v", got, tt.want) } + }) + } +} + +// TestParseKMSesURLConformance tests that all config attrs whose values can be +// URLs, IP addresses, or host:port addresses, when configured with an IPv6 +// address, the normalized to be conformant with RFC-5942 §4 +// See: https://rfc-editor.org/rfc/rfc5952.html +func TestParseKMSesURLConformance(t *testing.T) { + t.Parallel() + + for name, tc := range map[string]struct { + config string + expected map[string]string + shouldFail bool + }{ + "alicloudkms ipv4": { + config: ` +seal "alicloudkms" { + region = "us-east-1" + domain = "kms.us-east-1.aliyuncs.com" + access_key = "0wNEpMMlzy7szvai" + secret_key = "PupkTg8jdmau1cXxYacgE736PJj4cA" + kms_key_id = "08c33a6f-4e0a-4a1b-a3fa-7ddfa1d4fb73" +}`, + expected: map[string]string{ + "region": "us-east-1", + "domain": "kms.us-east-1.aliyuncs.com", + "access_key": "0wNEpMMlzy7szvai", + "secret_key": "PupkTg8jdmau1cXxYacgE736PJj4cA", + "kms_key_id": "08c33a6f-4e0a-4a1b-a3fa-7ddfa1d4fb73", + }, + }, + "alicloudkms ipv6": { + config: ` +seal "alicloudkms" { + region = "us-east-1" + domain = "2001:db8:0:0:0:0:2:1" + access_key = "0wNEpMMlzy7szvai" + secret_key = "PupkTg8jdmau1cXxYacgE736PJj4cA" + kms_key_id = "08c33a6f-4e0a-4a1b-a3fa-7ddfa1d4fb73" +}`, + expected: map[string]string{ + "region": "us-east-1", + "domain": "2001:db8::2:1", + "access_key": "0wNEpMMlzy7szvai", + "secret_key": "PupkTg8jdmau1cXxYacgE736PJj4cA", + "kms_key_id": "08c33a6f-4e0a-4a1b-a3fa-7ddfa1d4fb73", + }, + }, + "awskms ipv4": { + config: ` +seal "awskms" { + region = "us-east-1" + access_key = "AKIAIOSFODNN7EXAMPLE" + secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + kms_key_id = "19ec80b0-dfdd-4d97-8164-c6examplekey" + endpoint = "https://vpce-0e1bb1852241f8cc6-pzi0do8n.kms.us-east-1.vpce.amazonaws.com" +}`, + expected: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "kms_key_id": "19ec80b0-dfdd-4d97-8164-c6examplekey", + "endpoint": "https://vpce-0e1bb1852241f8cc6-pzi0do8n.kms.us-east-1.vpce.amazonaws.com", + }, + }, + "awskms ipv6": { + config: ` +seal "awskms" { + region = "us-east-1" + access_key = "AKIAIOSFODNN7EXAMPLE" + secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + kms_key_id = "19ec80b0-dfdd-4d97-8164-c6examplekey" + endpoint = "https://[2001:db8:0:0:0:0:2:1]:5984/my-aws-endpoint" +}`, + expected: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "kms_key_id": "19ec80b0-dfdd-4d97-8164-c6examplekey", + "endpoint": "https://[2001:db8::2:1]:5984/my-aws-endpoint", + }, + }, + "azurekeyvault ipv4": { + config: ` +seal "azurekeyvault" { + tenant_id = "46646709-b63e-4747-be42-516edeaf1e14" + client_id = "03dc33fc-16d9-4b77-8152-3ec568f8af6e" + client_secret = "DUJDS3..." + vault_name = "hc-vault" + key_name = "vault_key" + resource = "vault.azure.net" +}`, + expected: map[string]string{ + "tenant_id": "46646709-b63e-4747-be42-516edeaf1e14", + "client_id": "03dc33fc-16d9-4b77-8152-3ec568f8af6e", + "client_secret": "DUJDS3...", + "vault_name": "hc-vault", + "key_name": "vault_key", + "resource": "vault.azure.net", + }, + }, + "azurekeyvault ipv6": { + config: ` +seal "azurekeyvault" { + tenant_id = "46646709-b63e-4747-be42-516edeaf1e14" + client_id = "03dc33fc-16d9-4b77-8152-3ec568f8af6e" + client_secret = "DUJDS3..." + vault_name = "hc-vault" + key_name = "vault_key" + resource = "2001:db8:0:0:0:0:2:1", +}`, + expected: map[string]string{ + "tenant_id": "46646709-b63e-4747-be42-516edeaf1e14", + "client_id": "03dc33fc-16d9-4b77-8152-3ec568f8af6e", + "client_secret": "DUJDS3...", + "vault_name": "hc-vault", + "key_name": "vault_key", + "resource": "2001:db8::2:1", + }, + }, + "ocikms ipv4": { + config: ` +seal "ocikms" { + key_id = "ocid1.key.oc1.iad.afnxza26aag4s.abzwkljsbapzb2nrha5nt3s7s7p42ctcrcj72vn3kq5qx" + crypto_endpoint = "https://afnxza26aag4s-crypto.kms.us-ashburn-1.oraclecloud.com" + management_endpoint = "https://afnxza26aag4s-management.kms.us-ashburn-1.oraclecloud.com" + auth_type_api_key = "true" +}`, + expected: map[string]string{ + "key_id": "ocid1.key.oc1.iad.afnxza26aag4s.abzwkljsbapzb2nrha5nt3s7s7p42ctcrcj72vn3kq5qx", + "crypto_endpoint": "https://afnxza26aag4s-crypto.kms.us-ashburn-1.oraclecloud.com", + "management_endpoint": "https://afnxza26aag4s-management.kms.us-ashburn-1.oraclecloud.com", + "auth_type_api_key": "true", + }, + }, + "ocikms ipv6": { + config: ` +seal "ocikms" { + key_id = "https://[2001:db8:0:0:0:0:2:1]/abzwkljsbapzb2nrha5nt3s7s7p42ctcrcj72vn3kq5qx" + crypto_endpoint = "https://[2001:db8:0:0:0:0:2:1]/afnxza26aag4s-crypto" + management_endpoint = "https://[2001:db8:0:0:0:0:2:1]/afnxza26aag4s-management" + auth_type_api_key = "true" +}`, + expected: map[string]string{ + "key_id": "https://[2001:db8::2:1]/abzwkljsbapzb2nrha5nt3s7s7p42ctcrcj72vn3kq5qx", + "crypto_endpoint": "https://[2001:db8::2:1]/afnxza26aag4s-crypto", + "management_endpoint": "https://[2001:db8::2:1]/afnxza26aag4s-management", + "auth_type_api_key": "true", + }, + }, + "transit ipv4": { + config: ` +seal "transit" { + address = "https://vault:8200" + token = "s.Qf1s5zigZ4OX6akYjQXJC1jY" + disable_renewal = "false" + key_name = "transit_key_name" + mount_path = "transit/" + namespace = "ns1/" +} +`, + expected: map[string]string{ + "address": "https://vault:8200", + "token": "s.Qf1s5zigZ4OX6akYjQXJC1jY", + "disable_renewal": "false", + "key_name": "transit_key_name", + "mount_path": "transit/", + "namespace": "ns1/", + }, + }, + "transit ipv6": { + config: ` +seal "transit" { + address = "https://[2001:db8:0:0:0:0:2:1]:8200" + token = "s.Qf1s5zigZ4OX6akYjQXJC1jY" + disable_renewal = "false" + key_name = "transit_key_name" + mount_path = "transit/" + namespace = "ns1/" +} +`, + expected: map[string]string{ + "address": "https://[2001:db8::2:1]:8200", + "token": "s.Qf1s5zigZ4OX6akYjQXJC1jY", + "disable_renewal": "false", + "key_name": "transit_key_name", + "mount_path": "transit/", + "namespace": "ns1/", + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + kmses, err := ParseKMSes(tc.config) + require.NoError(t, err) + require.Len(t, kmses, 1) + require.EqualValues(t, tc.expected, kmses[0].Config) + }) + } +} + +// TestMergeKMSEnvConfigAddrConformance tests that all env config whose values +// can be URLs, IP addresses, or host:port addresses, when configured with an +// an IPv6 address, the normalized to be conformant with RFC-5942 §4 +// See: https://rfc-editor.org/rfc/rfc5952.html +func TestMergeKMSEnvConfigAddrConformance(t *testing.T) { + for name, tc := range map[string]struct { + sealType string // default to name if none given + kmsConfig map[string]string + envVars map[string]string + expected map[string]string + }{ + "alicloudkms": { + kmsConfig: map[string]string{ + "region": "us-east-1", + "domain": "kms.us-east-1.aliyuncs.com", + "access_key": "0wNEpMMlzy7szvai", + "secret_key": "PupkTg8jdmau1cXxYacgE736PJj4cA", + "kms_key_id": "08c33a6f-4e0a-4a1b-a3fa-7ddfa1d4fb73", + }, + envVars: map[string]string{"ALICLOUD_DOMAIN": "2001:db8:0:0:0:0:2:1"}, + expected: map[string]string{ + "region": "us-east-1", + "domain": "2001:db8::2:1", + "access_key": "0wNEpMMlzy7szvai", + "secret_key": "PupkTg8jdmau1cXxYacgE736PJj4cA", + "kms_key_id": "08c33a6f-4e0a-4a1b-a3fa-7ddfa1d4fb73", + }, + }, + "awskms": { + kmsConfig: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "kms_key_id": "19ec80b0-dfdd-4d97-8164-c6examplekey", + "endpoint": "https://vpce-0e1bb1852241f8cc6-pzi0do8n.kms.us-east-1.vpce.amazonaws.com", + }, + envVars: map[string]string{"AWS_KMS_ENDPOINT": "https://[2001:db8:0:0:0:0:2:1]:5984/my-aws-endpoint"}, + expected: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "kms_key_id": "19ec80b0-dfdd-4d97-8164-c6examplekey", + "endpoint": "https://[2001:db8::2:1]:5984/my-aws-endpoint", + }, + }, + "azurekeyvault": { + kmsConfig: map[string]string{ + "tenant_id": "46646709-b63e-4747-be42-516edeaf1e14", + "client_id": "03dc33fc-16d9-4b77-8152-3ec568f8af6e", + "client_secret": "DUJDS3...", + "vault_name": "hc-vault", + "key_name": "vault_key", + "resource": "vault.azure.net", + }, + envVars: map[string]string{"AZURE_AD_RESOURCE": "2001:db8:0:0:0:0:2:1"}, + expected: map[string]string{ + "tenant_id": "46646709-b63e-4747-be42-516edeaf1e14", + "client_id": "03dc33fc-16d9-4b77-8152-3ec568f8af6e", + "client_secret": "DUJDS3...", + "vault_name": "hc-vault", + "key_name": "vault_key", + "resource": "2001:db8::2:1", + }, + }, + "ocikms wrapper env vars": { + sealType: "ocikms", + kmsConfig: map[string]string{ + "key_id": "ocid1.key.oc1.iad.afnxza26aag4s.abzwkljsbapzb2nrha5nt3s7s7p42ctcrcj72vn3kq5qx", + "crypto_endpoint": "https://afnxza26aag4s-crypto.kms.us-ashburn-1.oraclecloud.com", + "management_endpoint": "https://afnxza26aag4s-management.kms.us-ashburn-1.oraclecloud.com", + "auth_type_api_key": "true", + }, + envVars: map[string]string{ + ocikms.EnvOciKmsWrapperKeyId: "https://[2001:db8:0:0:0:0:2:1]/abzwkljsbapzb2nrha5nt3s7s7p42ctcrcj72vn3kq5qx", + ocikms.EnvOciKmsWrapperCryptoEndpoint: "https://[2001:db8:0:0:0:0:2:1]/afnxza26aag4s-crypto", + ocikms.EnvOciKmsWrapperManagementEndpoint: "https://[2001:db8:0:0:0:0:2:1]/afnxza26aag4s-management", + }, + expected: map[string]string{ + "key_id": "https://[2001:db8::2:1]/abzwkljsbapzb2nrha5nt3s7s7p42ctcrcj72vn3kq5qx", + "crypto_endpoint": "https://[2001:db8::2:1]/afnxza26aag4s-crypto", + "management_endpoint": "https://[2001:db8::2:1]/afnxza26aag4s-management", + "auth_type_api_key": "true", + }, + }, + "ocikms vault env vars": { + sealType: "ocikms", + kmsConfig: map[string]string{ + "key_id": "ocid1.key.oc1.iad.afnxza26aag4s.abzwkljsbapzb2nrha5nt3s7s7p42ctcrcj72vn3kq5qx", + "crypto_endpoint": "https://afnxza26aag4s-crypto.kms.us-ashburn-1.oraclecloud.com", + "management_endpoint": "https://afnxza26aag4s-management.kms.us-ashburn-1.oraclecloud.com", + "auth_type_api_key": "true", + }, + envVars: map[string]string{ + ocikms.EnvVaultOciKmsSealKeyId: "https://[2001:db8:0:0:0:0:2:1]/abzwkljsbapzb2nrha5nt3s7s7p42ctcrcj72vn3kq5qx", + ocikms.EnvVaultOciKmsSealCryptoEndpoint: "https://[2001:db8:0:0:0:0:2:1]/afnxza26aag4s-crypto", + ocikms.EnvVaultOciKmsSealManagementEndpoint: "https://[2001:db8:0:0:0:0:2:1]/afnxza26aag4s-management", + }, + expected: map[string]string{ + "key_id": "https://[2001:db8::2:1]/abzwkljsbapzb2nrha5nt3s7s7p42ctcrcj72vn3kq5qx", + "crypto_endpoint": "https://[2001:db8::2:1]/afnxza26aag4s-crypto", + "management_endpoint": "https://[2001:db8::2:1]/afnxza26aag4s-management", + "auth_type_api_key": "true", + }, + }, + "transit addr not in config": { + sealType: "transit", + kmsConfig: map[string]string{ + "token": "s.Qf1s5zigZ4OX6akYjQXJC1jY", + "disable_renewal": "false", + "key_name": "transit_key_name", + "mount_path": "transit/", + "namespace": "ns1/", + }, + envVars: map[string]string{"VAULT_ADDR": "https://[2001:db8:0:0:0:0:2:1]:8200"}, + expected: map[string]string{ + // NOTE: If our address has not been configured we'll fall back to VAULT_ADDR for transit. + "address": "https://[2001:db8::2:1]:8200", + "token": "s.Qf1s5zigZ4OX6akYjQXJC1jY", + "disable_renewal": "false", + "key_name": "transit_key_name", + "mount_path": "transit/", + "namespace": "ns1/", + }, + }, + "transit addr in config": { + sealType: "transit", + kmsConfig: map[string]string{ + "address": "https://vault:8200", + "token": "s.Qf1s5zigZ4OX6akYjQXJC1jY", + "disable_renewal": "false", + "key_name": "transit_key_name", + "mount_path": "transit/", + "namespace": "ns1/", + }, + envVars: map[string]string{"VAULT_ADDR": "https://[2001:db8:0:0:0:0:2:1]:8200"}, + expected: map[string]string{ + // NOTE: If our address has been configured we don't consider VAULT_ADDR + "address": "https://vault:8200", + "token": "s.Qf1s5zigZ4OX6akYjQXJC1jY", + "disable_renewal": "false", + "key_name": "transit_key_name", + "mount_path": "transit/", + "namespace": "ns1/", + }, + }, + } { + t.Run(name, func(t *testing.T) { + typ := name + if tc.sealType != "" { + typ = tc.sealType + } + kms := &KMS{ + Type: typ, + Config: tc.kmsConfig, + } - for env := range tt.envVars { - if err := os.Unsetenv(env); err != nil { - t.Errorf("error unsetting environment vars for test: %s", err) - } + for envName, envVal := range tc.envVars { + t.Setenv(envName, envVal) } + + require.NoError(t, mergeKMSEnvConfig(kms)) + require.EqualValues(t, tc.expected, kms.Config) }) } } diff --git a/internalshared/configutil/listener.go b/internalshared/configutil/listener.go index b9ed168abf7e..d06a70bdcc70 100644 --- a/internalshared/configutil/listener.go +++ b/internalshared/configutil/listener.go @@ -177,7 +177,7 @@ func (l *Listener) Validate(path string) []ConfigError { func ParseSingleIPTemplate(ipTmpl string) (string, error) { r := regexp.MustCompile("{{.*?}}") if !r.MatchString(ipTmpl) { - return ipTmpl, nil + return NormalizeAddr(ipTmpl), nil } out, err := template.Parse(ipTmpl) diff --git a/internalshared/configutil/listener_test.go b/internalshared/configutil/listener_test.go index 51d0c094ed3b..02237ca45b7f 100644 --- a/internalshared/configutil/listener_test.go +++ b/internalshared/configutil/listener_test.go @@ -16,17 +16,49 @@ import ( // ensure that we only attempt to parse templates when the input contains a // template placeholder (see: go-sockaddr/template). func TestListener_ParseSingleIPTemplate(t *testing.T) { + t.Parallel() + tests := map[string]struct { arg string want string isErrorExpected bool errorMessage string }{ - "test https addr": { + "test hostname": { arg: "https://vaultproject.io:8200", want: "https://vaultproject.io:8200", isErrorExpected: false, }, + "test ipv4": { + arg: "https://10.10.1.10:8200", + want: "https://10.10.1.10:8200", + isErrorExpected: false, + }, + "test ipv6 RFC-5952 4.1 conformance leading zeroes": { + arg: "https://[2001:0db8::0001]:8200", + want: "https://[2001:db8::1]:8200", + isErrorExpected: false, + }, + "test ipv6 RFC-5952 4.2.2 conformance one 16-bit 0 field": { + arg: "https://[2001:db8:0:1:1:1:1:1]:8200", + want: "https://[2001:db8:0:1:1:1:1:1]:8200", + isErrorExpected: false, + }, + "test ipv6 RFC-5952 4.2.3 conformance longest run of 0 bits shortened": { + arg: "https://[2001:0:0:1:0:0:0:1]:8200", + want: "https://[2001:0:0:1::1]:8200", + isErrorExpected: false, + }, + "test ipv6 RFC-5952 4.2.3 conformance equal runs of 0 bits shortened": { + arg: "https://[2001:db8:0:0:1:0:0:1]:8200", + want: "https://[2001:db8::1:0:0:1]:8200", + isErrorExpected: false, + }, + "test ipv6 RFC-5952 4.3 conformance downcase hex letters": { + arg: "https://[2001:DB8:AC3:FE4::1]:8200", + want: "https://[2001:db8:ac3:fe4::1]:8200", + isErrorExpected: false, + }, "test invalid template func": { arg: "{{ FooBar }}", want: "", @@ -43,6 +75,7 @@ func TestListener_ParseSingleIPTemplate(t *testing.T) { name := name tc := tc t.Run(name, func(t *testing.T) { + t.Parallel() got, err := ParseSingleIPTemplate(tc.arg) if tc.isErrorExpected { diff --git a/internalshared/configutil/normalize.go b/internalshared/configutil/normalize.go new file mode 100644 index 000000000000..902699e57bfd --- /dev/null +++ b/internalshared/configutil/normalize.go @@ -0,0 +1,91 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package configutil + +import ( + "fmt" + "net" + "net/url" + "strings" +) + +// NormalizeAddr takes an address as a string and returns a normalized copy. +// If the addr is a URL, IP Address, or host:port address that includes an IPv6 +// address, the normalized copy will be conformant with RFC-5942 §4 +// See: https://rfc-editor.org/rfc/rfc5952.html +func NormalizeAddr(address string) string { + if address == "" { + return "" + } + + var ip net.IP + var port string + bracketedIPv6 := false + + // Try parsing it as a URL + pu, err := url.Parse(address) + if err == nil { + // We've been given something that appears to be a URL. See if the hostname + // is an IP address + ip = net.ParseIP(pu.Hostname()) + } else { + // We haven't been given a URL. Try and parse it as an IP address + ip = net.ParseIP(address) + if ip == nil { + // We haven't been given a URL or IP address, try parsing an IP:Port + // combination. + idx := strings.LastIndex(address, ":") + if idx > 0 { + // We've perhaps received an IP:Port address + addr := address[:idx] + port = address[idx+1:] + if strings.HasPrefix(addr, "[") && strings.HasSuffix(addr, "]") { + addr = strings.TrimPrefix(strings.TrimSuffix(addr, "]"), "[") + bracketedIPv6 = true + } + ip = net.ParseIP(addr) + } + } + } + + // If our IP is nil whatever was passed in does not contain an IP address. + if ip == nil { + return address + } + + if v4 := ip.To4(); v4 != nil { + return address + } + + if v6 := ip.To16(); v6 != nil { + // net.IP String() will return IPv6 RFC-5952 conformant addresses. + + if pu != nil { + // Return the URL in conformant fashion + if port := pu.Port(); port != "" { + pu.Host = fmt.Sprintf("[%s]:%s", v6.String(), port) + } else { + pu.Host = fmt.Sprintf("[%s]", v6.String()) + } + return pu.String() + } + + // Handle IP:Port addresses + if port != "" { + // Return the address:port or [address]:port + if bracketedIPv6 { + return fmt.Sprintf("[%s]:%s", v6.String(), port) + } else { + return fmt.Sprintf("%s:%s", v6.String(), port) + } + } + + // Handle just an IP address + return v6.String() + } + + // It shouldn't be possible to get to this point. If we somehow we manage + // to, return the string unchanged. + return address +} diff --git a/internalshared/configutil/normalize_test.go b/internalshared/configutil/normalize_test.go new file mode 100644 index 000000000000..d1aec31e621b --- /dev/null +++ b/internalshared/configutil/normalize_test.go @@ -0,0 +1,96 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package configutil + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestNormalizeAddr ensures that strings that match either an IP address or URL +// and contain an IPv6 address conform to RFC-5942 §4 +// See: https://rfc-editor.org/rfc/rfc5952.html +func TestNormalizeAddr(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + addr string + expected string + isErrorExpected bool + }{ + "hostname": { + addr: "https://vaultproject.io:8200", + expected: "https://vaultproject.io:8200", + }, + "ipv4": { + addr: "10.10.1.10", + expected: "10.10.1.10", + }, + "ipv4 IP:Port addr": { + addr: "10.10.1.10:8500", + expected: "10.10.1.10:8500", + }, + "ipv4 URL": { + addr: "https://10.10.1.10:8200", + expected: "https://10.10.1.10:8200", + }, + "ipv6 IP:Port addr no brackets": { + addr: "2001:0db8::0001:8500", + expected: "2001:db8::1:8500", + }, + "ipv6 IP:Port addr with brackets": { + addr: "[2001:0db8::0001]:8500", + expected: "[2001:db8::1]:8500", + }, + "ipv6 RFC-5952 4.1 conformance leading zeroes": { + addr: "2001:0db8::0001", + expected: "2001:db8::1", + }, + "ipv6 URL RFC-5952 4.1 conformance leading zeroes": { + addr: "https://[2001:0db8::0001]:8200", + expected: "https://[2001:db8::1]:8200", + }, + "ipv6 RFC-5952 4.2.2 conformance one 16-bit 0 field": { + addr: "2001:db8:0:1:1:1:1:1", + expected: "2001:db8:0:1:1:1:1:1", + }, + "ipv6 URL RFC-5952 4.2.2 conformance one 16-bit 0 field": { + addr: "https://[2001:db8:0:1:1:1:1:1]:8200", + expected: "https://[2001:db8:0:1:1:1:1:1]:8200", + }, + "ipv6 RFC-5952 4.2.3 conformance longest run of 0 bits shortened": { + addr: "2001:0:0:1:0:0:0:1", + expected: "2001:0:0:1::1", + }, + "ipv6 URL RFC-5952 4.2.3 conformance longest run of 0 bits shortened": { + addr: "https://[2001:0:0:1:0:0:0:1]:8200", + expected: "https://[2001:0:0:1::1]:8200", + }, + "ipv6 RFC-5952 4.2.3 conformance equal runs of 0 bits shortened": { + addr: "2001:db8:0:0:1:0:0:1", + expected: "2001:db8::1:0:0:1", + }, + "ipv6 URL RFC-5952 4.2.3 conformance equal runs of 0 bits shortened": { + addr: "https://[2001:db8:0:0:1:0:0:1]:8200", + expected: "https://[2001:db8::1:0:0:1]:8200", + }, + "ipv6 RFC-5952 4.3 conformance downcase hex letters": { + addr: "2001:DB8:AC3:FE4::1", + expected: "2001:db8:ac3:fe4::1", + }, + "ipv6 URL RFC-5952 4.3 conformance downcase hex letters": { + addr: "https://[2001:DB8:AC3:FE4::1]:8200", + expected: "https://[2001:db8:ac3:fe4::1]:8200", + }, + } + for name, tc := range tests { + name := name + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + require.Equal(t, tc.expected, NormalizeAddr(tc.addr)) + }) + } +} diff --git a/internalshared/configutil/telemetry.go b/internalshared/configutil/telemetry.go index 7c49fce00917..efa6adb2f4e9 100644 --- a/internalshared/configutil/telemetry.go +++ b/internalshared/configutil/telemetry.go @@ -34,7 +34,7 @@ const ( NumLeaseMetricsTimeBucketsDefault = 168 ) -// Telemetry is the telemetry configuration for the server +// Telemetry is the telemetry configuration for the server. type Telemetry struct { FoundKeys []string `hcl:",decodedFields"` UnusedKeys UnusedKeyMap `hcl:",unusedKeyPositions"` @@ -192,6 +192,11 @@ func parseTelemetry(result *SharedConfig, list *ast.ObjectList) error { return multierror.Prefix(err, "telemetry:") } + // Make sure addresses conform to RFC-5942 §4. If you've added new fields that + // are an address or URL be sure to update normalizeTelemetryAddresses(). + // See: https://rfc-editor.org/rfc/rfc5952.html + normalizeTelemetryAddresses(result.Telemetry) + if result.Telemetry.PrometheusRetentionTimeRaw != nil { var err error if result.Telemetry.PrometheusRetentionTime, err = parseutil.ParseDurationSecond(result.Telemetry.PrometheusRetentionTimeRaw); err != nil { @@ -241,6 +246,33 @@ func parseTelemetry(result *SharedConfig, list *ast.ObjectList) error { return nil } +// normalizeTelemetryAddresses ensures that any telemetry configuration that can +// be a URL, IP Address, or host:port address is conformant with RFC-5942 §4 +// See: https://rfc-editor.org/rfc/rfc5952.html +func normalizeTelemetryAddresses(in *Telemetry) { + if in == nil { + return + } + + // Make sure addresses conform to RFC-5952 + for _, addr := range []*string{ + &in.CirconusAPIURL, + &in.CirconusCheckSubmissionURL, + &in.DogStatsDAddr, + &in.StatsdAddr, + &in.StatsiteAddr, + } { + if addr == nil { + continue + } + + if url := *addr; url != "" { + n := NormalizeAddr(url) + *addr = n + } + } +} + type SetupTelemetryOpts struct { Config *Telemetry Ui cli.Ui diff --git a/internalshared/configutil/telemetry_test.go b/internalshared/configutil/telemetry_test.go index 285278eeaeba..ae2fd1f33cb9 100644 --- a/internalshared/configutil/telemetry_test.go +++ b/internalshared/configutil/telemetry_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestParsePrefixFilters(t *testing.T) { @@ -54,3 +55,47 @@ func TestParsePrefixFilters(t *testing.T) { } }) } + +// TestNormalizeTelemetryAddresses ensures that any telemetry configuration that +// can be a URL, IP Address, or host:port address is conformant with RFC-5942 §4 +// See: https://rfc-editor.org/rfc/rfc5952.html +func TestNormalizeTelemetryAddresses(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + given *Telemetry + expected *Telemetry + }{ + "ipv6-conformance": { + given: &Telemetry{ + // RFC-5952 4.1 leading zeroes + CirconusAPIURL: "https://[2001:0db8::0001]:443", + // RFC-5952 4.2.3 longest run of 0 bits shortened + CirconusCheckSubmissionURL: "https://[2001:0:0:1:0:0:0:1]:443", + // RFC-5952 4.2.3 equal runs of 0 bits shortened + DogStatsDAddr: "https://[2001:db8:0:0:1:0:0:1]:443", + // RFC-5952 4.3 downcase hex letters + StatsdAddr: "https://[2001:DB8:AC3:FE4::1]:443", + StatsiteAddr: "https://[2001:DB8:AC3:FE4::1]:443", + }, + expected: &Telemetry{ + CirconusAPIURL: "https://[2001:db8::1]:443", + CirconusCheckSubmissionURL: "https://[2001:0:0:1::1]:443", + DogStatsDAddr: "https://[2001:db8::1:0:0:1]:443", + StatsdAddr: "https://[2001:db8:ac3:fe4::1]:443", + StatsiteAddr: "https://[2001:db8:ac3:fe4::1]:443", + }, + }, + } + + for name, tc := range tests { + name := name + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + normalizeTelemetryAddresses(tc.given) + require.EqualValues(t, tc.expected, tc.given) + }) + } +} diff --git a/vault/identity_store_util.go b/vault/identity_store_util.go index 9418b41d6fff..3b7938114de1 100644 --- a/vault/identity_store_util.go +++ b/vault/identity_store_util.go @@ -18,10 +18,6 @@ import ( memdb "github.com/hashicorp/go-memdb" "github.com/hashicorp/go-secure-stdlib/strutil" uuid "github.com/hashicorp/go-uuid" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/anypb" - "google.golang.org/protobuf/types/known/timestamppb" - "github.com/hashicorp/vault/helper/activationflags" "github.com/hashicorp/vault/helper/identity" "github.com/hashicorp/vault/helper/identity/mfa" @@ -29,6 +25,9 @@ import ( "github.com/hashicorp/vault/helper/storagepacker" "github.com/hashicorp/vault/sdk/helper/consts" "github.com/hashicorp/vault/sdk/logical" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/timestamppb" ) var (