Add autoscale test to E2E (#2096)

* initial attempt at autoscale test

* working autoscale test

* only add add’l options if passed in

* Adding 3 replicas for load tester deployment

* wait longer and linux only

* skip autoscale test for v1.9 clusters

#2114

Author: jackfrancis

test/e2e/kubernetes/deployment/deployment.go

@@ -55,9 +55,32 @@ type Container struct {
 
 // CreateLinuxDeploy will create a deployment for a given image with a name in a namespace
 // --overrides='{ "apiVersion": "extensions/v1beta1", "spec":{"template":{"spec": {"nodeSelector":{"beta.kubernetes.io/os":"linux"}}}}}'
-func CreateLinuxDeploy(image, name, namespace string) (*Deployment, error) {
+func CreateLinuxDeploy(image, name, namespace, miscOpts string) (*Deployment, error) {
+	var err error
+	var out []byte
 	overrides := `{ "apiVersion": "extensions/v1beta1", "spec":{"template":{"spec": {"nodeSelector":{"beta.kubernetes.io/os":"linux"}}}}}`
-	out, err := exec.Command("kubectl", "run", name, "-n", namespace, "--image", image, "--overrides", overrides).CombinedOutput()
+	if miscOpts != "" {
+		out, err = exec.Command("kubectl", "run", name, "-n", namespace, "--image", image, "--overrides", overrides, miscOpts).CombinedOutput()
+	} else {
+		out, err = exec.Command("kubectl", "run", name, "-n", namespace, "--image", image, "--overrides", overrides).CombinedOutput()
+	}
+	if err != nil {
+		log.Printf("Error trying to deploy %s [%s] in namespace %s:%s\n", name, image, namespace, string(out))
+		return nil, err
+	}
+	d, err := Get(name, namespace)
+	if err != nil {
+		log.Printf("Error while trying to fetch Deployment %s in namespace %s:%s\n", name, namespace, err)
+		return nil, err
+	}
+	return d, nil
+}
+
+// RunLinuxDeploy will create a deployment that runs a bash command in a pod
+// --overrides='{ "apiVersion": "extensions/v1beta1", "spec":{"template":{"spec": {"nodeSelector":{"beta.kubernetes.io/os":"linux"}}}}}'
+func RunLinuxDeploy(image, name, namespace, command string, replicas int) (*Deployment, error) {
+	overrides := `{ "apiVersion": "extensions/v1beta1", "spec":{"template":{"spec": {"nodeSelector":{"beta.kubernetes.io/os":"linux"}}}}}`
+	out, err := exec.Command("kubectl", "run", name, "-n", namespace, "--image", image, "--replicas", strconv.Itoa(replicas), "--overrides", overrides, "--command", "--", "/bin/sh", "-c", command).CombinedOutput()
 	if err != nil {
 		log.Printf("Error trying to deploy %s [%s] in namespace %s:%s\n", name, image, namespace, string(out))
 		return nil, err
@@ -113,10 +136,10 @@ func (d *Deployment) Delete() error {
 }
 
 // Expose will create a load balancer and expose the deployment on a given port
-func (d *Deployment) Expose(targetPort, exposedPort int) error {
-	out, err := exec.Command("kubectl", "expose", "deployment", d.Metadata.Name, "--type", "LoadBalancer", "-n", d.Metadata.Namespace, "--target-port", strconv.Itoa(targetPort), "--port", strconv.Itoa(exposedPort)).CombinedOutput()
+func (d *Deployment) Expose(svcType string, targetPort, exposedPort int) error {
+	out, err := exec.Command("kubectl", "expose", "deployment", d.Metadata.Name, "--type", svcType, "-n", d.Metadata.Namespace, "--target-port", strconv.Itoa(targetPort), "--port", strconv.Itoa(exposedPort)).CombinedOutput()
 	if err != nil {
-		log.Printf("Error while trying to expose target port (%v) for deployment %s in namespace %s on port %v:%s\n", targetPort, d.Metadata.Name, d.Metadata.Namespace, exposedPort, string(out))
+		log.Printf("Error while trying to expose (%s) target port (%v) for deployment %s in namespace %s on port %v:%s\n", svcType, targetPort, d.Metadata.Name, d.Metadata.Namespace, exposedPort, string(out))
 		return err
 	}
 	return nil

test/e2e/kubernetes/kubernetes_test.go

@@ -176,18 +176,88 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
 	})
 
 	Describe("with a linux agent pool", func() {
+		It("should be able to autoscale", func() {
+			version, err := node.Version()
+			Expect(err).NotTo(HaveOccurred())
+			re := regexp.MustCompile("v1.9")
+			if eng.HasLinuxAgents() && re.FindString(version) == "" {
+				// Inspired by http://blog.kubernetes.io/2016/07/autoscaling-in-kubernetes.html
+				r := rand.New(rand.NewSource(time.Now().UnixNano()))
+				phpApacheName := fmt.Sprintf("php-apache-%s-%v", cfg.Name, r.Intn(99999))
+				phpApacheDeploy, err := deployment.CreateLinuxDeploy("gcr.io/google_containers/hpa-example", phpApacheName, "default", "--requests=cpu=50m,memory=50M")
+				if err != nil {
+					fmt.Println(err)
+				}
+				Expect(err).NotTo(HaveOccurred())
+
+				running, err := pod.WaitOnReady(phpApacheName, "default", 3, 30*time.Second, cfg.Timeout)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(running).To(Equal(true))
+
+				phpPods, err := phpApacheDeploy.Pods()
+				Expect(err).NotTo(HaveOccurred())
+				// We should have exactly 1 pod to begin
+				Expect(len(phpPods)).To(Equal(1))
+
+				err = phpApacheDeploy.Expose("ClusterIP", 80, 80)
+				Expect(err).NotTo(HaveOccurred())
+				s, err := service.Get(phpApacheName, "default")
+				Expect(err).NotTo(HaveOccurred())
+
+				// Apply autoscale characteristics to deployment
+				_, err = exec.Command("kubectl", "autoscale", "deployment", phpApacheName, "--cpu-percent=5", "--min=1", "--max=10").CombinedOutput()
+				Expect(err).NotTo(HaveOccurred())
+
+				phpPods, err = phpApacheDeploy.Pods()
+				Expect(err).NotTo(HaveOccurred())
+				// We should still have exactly 1 pod after autoscale config but before load
+				Expect(len(phpPods)).To(Equal(1))
+
+				// Launch a simple busybox pod that wget's continuously to the apache serviceto simulate load
+				commandString := fmt.Sprintf("while true; do wget -q -O- http://%s.default.svc.cluster.local; done", phpApacheName)
+				loadTestName := fmt.Sprintf("load-test-%s-%v", cfg.Name, r.Intn(99999))
+				numLoadTestPods := 3
+				loadTestDeploy, err := deployment.RunLinuxDeploy("busybox", loadTestName, "default", commandString, numLoadTestPods)
+				Expect(err).NotTo(HaveOccurred())
+
+				running, err = pod.WaitOnReady(loadTestName, "default", 3, 30*time.Second, cfg.Timeout)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(running).To(Equal(true))
+
+				// We should have three load tester pods running
+				loadTestPods, err := loadTestDeploy.Pods()
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(loadTestPods)).To(Equal(numLoadTestPods))
+
+				// Wait 3 minutes for autoscaler to respond to load
+				time.Sleep(3 * time.Minute)
+
+				phpPods, err = phpApacheDeploy.Pods()
+				Expect(err).NotTo(HaveOccurred())
+				// We should have > 1 pods after autoscale effects
+				Expect(len(phpPods) > 1).To(BeTrue())
+
+				err = loadTestDeploy.Delete()
+				Expect(err).NotTo(HaveOccurred())
+				err = phpApacheDeploy.Delete()
+				Expect(err).NotTo(HaveOccurred())
+				err = s.Delete()
+				Expect(err).NotTo(HaveOccurred())
+			}
+		})
+
 		It("should be able to deploy an nginx service", func() {
 			if eng.HasLinuxAgents() {
 				r := rand.New(rand.NewSource(time.Now().UnixNano()))
 				deploymentName := fmt.Sprintf("nginx-%s-%v", cfg.Name, r.Intn(99999))
-				nginxDeploy, err := deployment.CreateLinuxDeploy("library/nginx:latest", deploymentName, "default")
+				nginxDeploy, err := deployment.CreateLinuxDeploy("library/nginx:latest", deploymentName, "default", "")
 				Expect(err).NotTo(HaveOccurred())
 
 				running, err := pod.WaitOnReady(deploymentName, "default", 3, 30*time.Second, cfg.Timeout)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(running).To(Equal(true))
 
-				err = nginxDeploy.Expose(80, 80)
+				err = nginxDeploy.Expose("LoadBalancer", 80, 80)
 				Expect(err).NotTo(HaveOccurred())
 
 				s, err := service.Get(deploymentName, "default")
@@ -231,7 +301,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
 				Expect(err).NotTo(HaveOccurred())
 				Expect(running).To(Equal(true))
 
-				err = iisDeploy.Expose(80, 80)
+				err = iisDeploy.Expose("LoadBalancer", 80, 80)
 				Expect(err).NotTo(HaveOccurred())
 
 				s, err := service.Get(deploymentName, "default")