Etcd3 Support (#1564)

* Support multiple etcd versions

* Modify git ignore and removing keys generated by test

* Review fixes

* Switching to mirrors

* Consider empty version valid on vLabs, gets defaulted in generalized api model

* Changed etcd to ver 3.1.10

* unify test etcd version with allowed versions

Author: khenidak

.gitignore

@@ -24,4 +24,8 @@ test/acs-engine-test/report/TestReport.json
 *.swp
 
 # I have no idea why these get generated when I run the e2e test
-test/e2e/kubernetes/translations/ 
+test/e2e/kubernetes/translations/
+
+# test outputs
+cmd/_test_output
+

cmd/deploy_test.go

@@ -15,7 +15,7 @@ import (
 const ExampleAPIModel = `{
   "apiVersion": "vlabs",
   "properties": {
-    "orchestratorProfile": { "orchestratorType": "Kubernetes", "kubernetesConfig": { "useManagedIdentity": %s } },
+		"orchestratorProfile": { "orchestratorType": "Kubernetes", "kubernetesConfig": { "useManagedIdentity": %s, "etcdVersion" : "2.5.2" } },
     "masterProfile": { "count": 1, "dnsPrefix": "", "vmSize": "Standard_D2_v2" },
     "agentPoolProfiles": [ { "name": "linuxpool1", "count": 2, "vmSize": "Standard_D2_v2", "availabilityProfile": "AvailabilitySet" } ],
     "windowsProfile": { "adminUsername": "azureuser", "adminPassword": "replacepassword1234$" },

parts/kubernetesmaster-kube-apiserver.yaml

@@ -35,7 +35,7 @@ spec:
         - "--oidc-client-id="
         - "--oidc-issuer-url="
         - "--oidc-username-claim=oid"
-        - "--storage-backend=etcd2"
+        - "--storage-backend=<etcdApiVersion>"
         - "--v=4"
         - "<kubernetesEnableRbac>"
         - "--requestheader-allowed-names="

parts/kubernetesmastercustomdata.yml

@@ -1,7 +1,9 @@
 #cloud-config
 
 packages:
+{{if not .OrchestratorProfile.IsCustomEtcdVersion}}
  - etcd
+{{end}}
  - jq
  - traceroute
 
@@ -292,6 +294,7 @@ write_files:
     sed -i "/requestheader-group-headers/d" "/etc/kubernetes/manifests/kube-apiserver.yaml"
     sed -i "/requestheader-username-headers/d" "/etc/kubernetes/manifests/kube-apiserver.yaml"
 {{end}}
+    sed -i "s|<etcdApiVersion>|{{ .OrchestratorProfile.GetAPIServerEtcdAPIVersion }}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"
 
 - path: "/opt/azure/containers/provision.sh"
   permissions: "0744"
@@ -307,7 +310,57 @@ write_files:
   content: !!binary |
     {{WrapAsVariable "mountetcdScript"}}
 
+{{if .OrchestratorProfile.IsCustomEtcdVersion}}
+- path: "/etc/systemd/system/etcd.service"
+  permissions: "0644"
+  owner: "root"
+  content: |
+    [Unit]
+    Description=etcd - highly-available key value store
+    Documentation=https://github.com/coreos/etcd
+    Documentation=man:etcd
+    After=network.target
+    Wants=network-online.target
+    [Service]
+    Environment=DAEMON_ARGS=
+    Environment=ETCD_NAME=%H
+    Environment=ETCD_DATA_DIR=/var/lib/etcd/default
+    EnvironmentFile=-/etc/default/%p
+    Type=notify
+    User=etcd
+    PermissionsStartOnly=true
+    ExecStart=/usr/bin/etcd $DAEMON_ARGS
+    Restart=always
+    [Install]
+    WantedBy=multi-user.target
+    Alias=etcd.service
+
+- path: "/opt/azure/containers/setup-etcd.sh"
+  permissions: "0744"
+  owner: "root"
+  content: |
+    #!/bin/bash
+    # if provisioning using any etcd version other than 2.5.2
+    # We download it and set it up here 
+    set -x
+    ETCD_VER=v{{ .OrchestratorProfile.KubernetesConfig.EtcdVersion }}
+    DOWNLOAD_URL=https://acs-mirror.azureedge.net/github-coreos # we use mirror to github
+    mkdir -p /tmp/etcd-download
+    curl -L ${DOWNLOAD_URL}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+    tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /usr/bin/ --strip-components=1
+    #create etcd user
+    useradd -U "etcd"
+    usermod -p "$(head -c 32 /dev/urandom | base64)" "etcd"
+    passwd -u "$etcd" > /dev/null
+    # update systemctl and start service
+    systemctl daemon-reload 
+    systemctl enable etcd.service
+{{end}}
+
 runcmd: 
+{{ if .OrchestratorProfile.IsCustomEtcdVersion }}
+- /opt/azure/containers/setup-etcd.sh
+{{end}}
 - apt-mark hold walinuxagent {{GetKubernetesMasterPreprovisionYaml}}
 - /bin/echo DAEMON_ARGS=--name "{{WrapAsVerbatim "variables('masterVMNames')[copyIndex(variables('masterOffset'))]"}}" --initial-advertise-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --advertise-client-urls "{{WrapAsVerbatim "variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-client-urls "{{WrapAsVerbatim "concat(variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))], ',http://127.0.0.1:', variables('masterEtcdClientPort'))"}}" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster "{{WrapAsVerbatim "variables('masterEtcdClusterStates')[div(variables('masterCount'), 2)]"}} --data-dir "/var/lib/etcddisk"" --initial-cluster-state "new" | tee -a /etc/default/etcd
 - sudo /bin/chown -R etcd:etcd /var/lib/etcd/default

pkg/acsengine/defaults.go

@@ -247,6 +247,12 @@ func setOrchestratorDefaults(cs *api.ContainerService) {
 				a.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitBucket = DefaultKubernetesCloudProviderRateLimitBucket
 			}
 		}
+
+		// default etcd version
+		if "" == a.OrchestratorProfile.KubernetesConfig.EtcdVersion {
+			a.OrchestratorProfile.KubernetesConfig.EtcdVersion = "2.5.2"
+		}
+
 	} else if a.OrchestratorProfile.OrchestratorType == api.DCOS {
 		if a.OrchestratorProfile.DcosConfig == nil {
 			a.OrchestratorProfile.DcosConfig = &api.DcosConfig{}

pkg/acsengine/testdata/etcd-versions/kubernetes.json

@@ -0,0 +1,53 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "kubernetesConfig": {
+        "etcdVersion": "3.1.10"
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "masterdns1",
+      "vmSize": "Standard_D2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D2_v2",
+        "availabilityProfile": "AvailabilitySet"
+      },
+      {
+        "name": "agentpool2",
+        "count": 3,
+        "vmSize": "Standard_D2_v2",
+        "availabilityProfile": "AvailabilitySet"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": "ssh-rsa PUBLICKEY azureuser@linuxvm"
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "ServicePrincipalClientID",
+      "secret": "myServicePrincipalClientSecret"
+    },
+    "certificateProfile": {
+      "caCertificate": "caCertificate",
+      "apiServerCertificate": "apiServerCertificate",
+      "apiServerPrivateKey": "apiServerPrivateKey",
+      "clientCertificate": "clientCertificate",
+      "clientPrivateKey": "clientPrivateKey",
+      "kubeConfigCertificate": "kubeConfigCertificate",
+      "kubeConfigPrivateKey": "kubeConfigPrivateKey"
+    }
+  }
+}

pkg/api/converterfromapi.go

@@ -669,6 +669,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet
 	vlabs.EnableAggregatedAPIs = api.EnableAggregatedAPIs
 	vlabs.GCHighThreshold = api.GCHighThreshold
 	vlabs.GCLowThreshold = api.GCLowThreshold
+	vlabs.EtcdVersion = api.EtcdVersion
 }
 
 func convertMasterProfileToV20160930(api *MasterProfile, v20160930 *v20160930.MasterProfile) {

pkg/api/convertertoapi.go

@@ -616,6 +616,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
 	api.EnableAggregatedAPIs = vlabs.EnableAggregatedAPIs
 	api.GCHighThreshold = vlabs.GCHighThreshold
 	api.GCLowThreshold = vlabs.GCLowThreshold
+	api.EtcdVersion = vlabs.EtcdVersion
 }
 
 func convertV20160930MasterProfile(v20160930 *v20160930.MasterProfile, api *MasterProfile) {

pkg/api/types.go

@@ -9,6 +9,7 @@ import (
 	"github.com/Azure/acs-engine/pkg/api/v20170131"
 	"github.com/Azure/acs-engine/pkg/api/v20170701"
 	"github.com/Azure/acs-engine/pkg/api/vlabs"
+	"github.com/Masterminds/semver"
 )
 
 // TypeMeta describes an individual API model object
@@ -185,6 +186,7 @@ type KubernetesConfig struct {
 	EnableAggregatedAPIs             bool    `json:"enableAggregatedAPIs,omitempty"`
 	GCHighThreshold                  int     `json:"gchighthreshold,omitempty"`
 	GCLowThreshold                   int     `json:"gclowthreshold,omitempty"`
+	EtcdVersion                      string  `json:"etcdVersion,omitempty"`
 }
 
 // DcosConfig Configuration for DC/OS
@@ -533,3 +535,18 @@ func (o *OrchestratorProfile) IsVNETIntegrated() bool {
 func (p *Properties) HasAadProfile() bool {
 	return p.AADProfile != nil
 }
+
+// IsCustomEtcdVersion Checks if etcd version is NOT default 2.5.2
+func (o *OrchestratorProfile) IsCustomEtcdVersion() bool {
+	return "2.5.2" != o.KubernetesConfig.EtcdVersion
+}
+
+// GetAPIServerEtcdAPIVersion Used to set apiserver's etcdapi version
+func (o *OrchestratorProfile) GetAPIServerEtcdAPIVersion() string {
+	// if we are here, version has already been validated..
+	etcdversion, _ := semver.NewVersion(o.KubernetesConfig.EtcdVersion)
+	if 2 == etcdversion.Major() {
+		return "etcd2"
+	}
+	return "etcd3"
+}

pkg/api/vlabs/types.go

@@ -204,6 +204,7 @@ type KubernetesConfig struct {
 	EnableAggregatedAPIs             bool    `json:"enableAggregatedAPIs,omitempty"`
 	GCHighThreshold                  int     `json:"gchighthreshold,omitempty"`
 	GCLowThreshold                   int     `json:"gclowthreshold,omitempty"`
+	EtcdVersion                      string  `json:"etcdVersion,omitempty"`
 }
 
 // DcosConfig Configuration for DC/OS

pkg/api/vlabs/validate.go

@@ -16,13 +16,32 @@ import (
 var (
 	validate        *validator.Validate
 	keyvaultIDRegex *regexp.Regexp
+	// Any version has to be mirrored in https://acs-mirror.azureedge.net/github-coreos/etcd-v[Version]-linux-amd64.tar.gz
+	etcdValidVersions = [...]string{"2.5.2", "3.1.10"}
 )
 
 func init() {
 	validate = validator.New()
 	keyvaultIDRegex = regexp.MustCompile(`^/subscriptions/\S+/resourceGroups/\S+/providers/Microsoft.KeyVault/vaults/[^/\s]+$`)
 }
 
+func isValidEtcdVersion(etcdVersion string) error {
+	// Empty versions is defaulted to 2.5.2 on the generalized api model
+	// after vlabs validation. Empty "" version is a valid version for etcd
+	if "" == etcdVersion {
+		return nil
+	}
+	// We have a version set by user
+	validVersions := "" // a bag of valid versions
+	for _, ver := range etcdValidVersions {
+		if ver == etcdVersion {
+			return nil
+		}
+		validVersions = fmt.Sprintf("%s %s", validVersions, ver)
+	}
+	return fmt.Errorf("Invalid etcd version(%s), valid versions are%s", etcdVersion, validVersions)
+}
+
 // Validate implements APIObject
 func (o *OrchestratorProfile) Validate() error {
 	// Don't need to call validate.Struct(o)
@@ -498,6 +517,11 @@ func (a *KubernetesConfig) Validate(k8sVersion string) error {
 		}
 	}
 
+	// Validate that we have a valid etcd version
+	if e := isValidEtcdVersion(a.EtcdVersion); e != nil {
+		return e
+	}
+
 	return nil
 }