Run the upgrade and downgrade tests in the upgrade prow job

test/README.md

@@ -12,10 +12,10 @@ Before running the integration, please make sure you have installed
 install custom resource for operator or knative-serving installed in your
 cluster.
 
-Create a namespace called `operator-tests` if it is missing.
+Create a namespace called `knative-serving` if it is missing.
 
 ```bash
-kubectl create namespace operator-tests
+kubectl create namespace knative-serving
 ```
 
 To run all integration tests:

test/e2e-common.sh

@@ -17,16 +17,20 @@
 # This script provides helper methods to perform cluster actions.
 source $(dirname $0)/../vendor/knative.dev/test-infra/scripts/e2e-tests.sh
 
-# Latest serving release. This is intentionally hardcoded for now, but
-# will need the ability to test against the latest successful serving
-# CI runs in the future.
-readonly LATEST_SERVING_RELEASE_VERSION=$(git describe --match "v[0-9]*" --abbrev=0)
+# Latest serving operator release.
+readonly LATEST_SERVING_OPERATOR_RELEASE_VERSION=$(git tag | sort -V | tail -1)
+# Latest serving release. This can be different from LATEST_SERVING_OPERATOR_RELEASE_VERSION.
+LATEST_SERVING_RELEASE_VERSION="v0.12.1"
 # Istio version we test with
 readonly ISTIO_VERSION="1.4.2"
 # Test without Istio mesh enabled
 readonly ISTIO_MESH=0
 # Namespace used for tests
-readonly TEST_NAMESPACE="operator-tests"
+readonly TEST_NAMESPACE="knative-serving"
+
+OPERATOR_DIR=$(dirname $0)/..
+KNATIVE_SERVING_DIR=${OPERATOR_DIR}/..
+release_yaml="$(mktemp)"
 
 # Choose a correct istio-crds.yaml file.
 # - $1 specifies Istio version.
@@ -50,6 +54,24 @@ function istio_yaml() {
   echo "third_party/istio-${istio_version}/istio-${suffix}.yaml"
 }
 
+# Download the repository of Knative Serving. The purpose of this function is to download the source code of serving
+# and retrive the LATEST_SERVING_RELEASE_VERSION for further use.
+# Parameter: $1 - branch of the repository.
+function donwload_knative_serving() {
+  # Go the directory to download the source code of knative serving
+  cd ${KNATIVE_SERVING_DIR}
+  # Download the source code of knative serving
+  git clone https://github.com/knative/serving.git
+  cd serving
+  local branch=$1
+  if [ -n "${branch}" ] ; then
+    git fetch origin ${branch}:${branch}
+    git checkout ${branch}
+  fi
+  LATEST_SERVING_RELEASE_VERSION=$(git tag | sort -V | tail -1)
+  cd ${OPERATOR_DIR}
+}
+
 # Install Istio.
 function install_istio() {
   local base_url="https://raw.githubusercontent.com/knative/serving/${LATEST_SERVING_RELEASE_VERSION}"
@@ -59,7 +81,7 @@ function install_istio() {
   echo ">> Installing Istio"
   echo "Istio CRD YAML: ${INSTALL_ISTIO_CRD_YAML}"
   echo "Istio YAML: ${INSTALL_ISTIO_YAML}"
-    
+
   echo ">> Bringing up Istio"
   echo ">> Running Istio CRD installer"
   kubectl apply -f "${INSTALL_ISTIO_CRD_YAML}" || return 1
@@ -76,6 +98,7 @@ function create_namespace() {
 }
 
 function install_serving_operator() {
+  cd ${OPERATOR_DIR}
   header "Installing Knative Serving operator"
   # Deploy the operator
   ko apply -f config/
@@ -87,7 +110,7 @@ function knative_teardown() {
   echo ">> Uninstalling Knative serving"
   echo "Istio YAML: ${INSTALL_ISTIO_YAML}"
   echo ">> Bringing down Serving"
-  kubectl delete -n knative-serving knativeserving --all
+  kubectl delete -n $TEST_NAMESPACE KnativeServing --all
   echo ">> Bringing down Istio"
   kubectl delete --ignore-not-found=true -f "${INSTALL_ISTIO_YAML}" || return 1
   kubectl delete --ignore-not-found=true clusterrolebinding cluster-admin-binding

test/e2e-tests.sh

@@ -29,6 +29,7 @@
 source $(dirname $0)/e2e-common.sh
 
 function knative_setup() {
+  donwload_knative_serving
   install_istio || fail_test "Istio installation failed"
   create_namespace
   install_serving_operator

test/e2e-upgrade-tests.sh

@@ -33,21 +33,21 @@
 
 source $(dirname $0)/e2e-common.sh
 
-OPERATOR_DIR=$(dirname $0)/..
-KNATIVE_SERVING_DIR=${OPERATOR_DIR}/..
+function install_previous_operator_release() {
+  local full_url="https://github.com/knative/serving-operator/releases/download/${LATEST_SERVING_OPERATOR_RELEASE_VERSION}/serving-operator.yaml"
 
-function install_latest_operator_release() {
-  header "Installing Knative Serving operator latest public release"
-  local full_url="https://github.com/knative/serving-operator/releases/download/${LATEST_SERVING_RELEASE_VERSION}/serving-operator.yaml"
-
-  local release_yaml="$(mktemp)"
   wget "${full_url}" -O "${release_yaml}" \
       || fail_test "Unable to download latest Knative Serving Operator release."
 
+  donwload_knative_serving
   install_istio || fail_test "Istio installation failed"
+  install_previous_serving_release
+}
+
+function install_previous_serving_release() {
+  header "Installing Knative Serving operator previous public release"
   kubectl apply -f "${release_yaml}" || fail_test "Knative Serving Operator latest release installation failed"
-  create_custom_resource
-  wait_until_pods_running ${TEST_NAMESPACE}
+  wait_until_pods_running default || fail_test "Serving Operator did not come up"
 }
 
 function create_custom_resource() {
@@ -73,31 +73,51 @@ EOF
 
 function knative_setup() {
   create_namespace
-  install_latest_operator_release
+  install_previous_operator_release
+  create_custom_resource
+  wait_until_pods_running ${TEST_NAMESPACE}
 }
 
-function install_head() {
+# Create test resources and images
+function test_setup() {
   generate_latest_serving_manifest
-  install_serving_operator
+  echo ">> Creating test resources (test/config/) in Knative Serving repository"
+  cd ${KNATIVE_SERVING_DIR}/serving
+  ko apply ${KO_FLAGS} -f test/config/ || return 1
+
+  echo ">> Uploading test images..."
+  # We only need to build and publish two images among all the test images
+  ${OPERATOR_DIR}/test/upload-test-images.sh ${KNATIVE_SERVING_DIR}/serving "test/test_images/pizzaplanetv1"
+  ${OPERATOR_DIR}/test/upload-test-images.sh ${KNATIVE_SERVING_DIR}/serving "test/test_images/pizzaplanetv2"
+
+  echo ">> Waiting for Ingress provider to be running..."
+  if [[ -n "${ISTIO_VERSION}" ]]; then
+    wait_until_pods_running istio-system || return 1
+    wait_until_service_has_external_ip istio-system istio-ingressgateway
+  fi
+  cd ${OPERATOR_DIR}
 }
 
+# This function either generate the manifest based on a branch or download the latest manifest for Knative Serving.
+# Parameter: $1 - branch name. If it is empty, download the manifest from nightly build.
 function generate_latest_serving_manifest() {
-  # Go the directory to download the source code of knative serving
-  cd ${KNATIVE_SERVING_DIR}
-
-  # Download the source code of knative serving
-  git clone https://github.com/knative/serving.git
-  cd serving
-  COMMIT_ID=$(git rev-parse --verify HEAD)
-  echo ">> The latest commit ID of Knative Serving is ${COMMIT_ID}."
+  cd ${KNATIVE_SERVING_DIR}/serving
   mkdir -p output
+  local branch=$1
+  if [[ -n "${branch}" ]]; then
+    git checkout ${branch}
+    COMMIT_ID=$(git rev-parse --verify HEAD)
+    echo ">> The latest commit ID of Knative Serving is ${COMMIT_ID}."
+    # Generate the manifest
+    export YAML_OUTPUT_DIR=${KNATIVE_SERVING_DIR}/serving/output
+    ./hack/generate-yamls.sh ${KNATIVE_SERVING_DIR}/serving ${YAML_OUTPUT_DIR}/output.yaml
+  else
+    echo ">> Download the latest nightly build of Knative Serving."
+    # Download the latest manifest
+    SERVING_YAML=${KNATIVE_SERVING_DIR}/serving/output/serving.yaml
+    wget -O ${SERVING_YAML} https://storage.googleapis.com/knative-nightly/serving/latest/serving.yaml
+  fi
 
-  # Generate the manifest
-  export YAML_OUTPUT_DIR=${KNATIVE_SERVING_DIR}/serving/output
-  ./hack/generate-yamls.sh ${KNATIVE_SERVING_DIR}/serving ${YAML_OUTPUT_DIR}/output.yaml
-
-  # Copy the serving.yaml into cmd/manager/kodata/knative-serving
-  SERVING_YAML=${KNATIVE_SERVING_DIR}/serving/output/serving.yaml
   if [[ -f "${SERVING_YAML}" ]]; then
     echo ">> Replacing the current manifest in operator with the generated manifest"
     rm -rf ${OPERATOR_DIR}/cmd/manager/kodata/knative-serving/*
@@ -115,14 +135,49 @@ initialize $@ --skip-istio-addon
 
 TIMEOUT=20m
 
-install_head
+header "Running preupgrade tests"
+
+cd ${KNATIVE_SERVING_DIR}/serving
+go_test_e2e -tags=preupgrade -timeout=${TIMEOUT} ./test/upgrade \
+  --resolvabledomain="false" "--https" || fail_test
+
+# Remove this in case we failed to clean it up in an earlier test.
+rm -f /tmp/prober-signal
+
+go_test_e2e -tags=probe -timeout=${TIMEOUT} ./test/upgrade \
+  --resolvabledomain="false" "--https" &
+PROBER_PID=$!
+echo "Prober PID is ${PROBER_PID}"
+
+install_serving_operator
 
 # If we got this far, the operator installed Knative Serving of the latest source code.
 header "Running tests for Knative Serving Operator"
 failed=0
 
-# Run the postupgrade tests
+# Run the postupgrade tests under operator
+# Operator tests here will make sure that all the Knative deployments reach the desired states and operator CR is
+# in ready state.
+cd ${OPERATOR_DIR}
 go_test_e2e -tags=postupgrade -timeout=${TIMEOUT} ./test/upgrade || failed=1
+wait_until_pods_running ${TEST_NAMESPACE}
+
+header "Running tests under Knative Serving"
+# Run the postupgrade tests under serving
+cd ${KNATIVE_SERVING_DIR}/serving
+go_test_e2e -tags=postupgrade -timeout=${TIMEOUT} ./test/upgrade || failed=1
+
+install_previous_serving_release
+wait_until_pods_running ${TEST_NAMESPACE}
+
+header "Running postdowngrade tests"
+go_test_e2e -tags=postdowngrade -timeout=${TIMEOUT} ./test/upgrade \
+  --resolvabledomain="false" || fail_test
+
+echo "done" > /tmp/prober-signal
+
+header "Waiting for prober test"
+wait ${PROBER_PID} || fail_test "Prober failed"
 
 # Require that tests succeeded.
 (( failed )) && fail_test

test/e2e_flags.go

@@ -23,7 +23,7 @@ import "os"
 
 var (
 	// ServingOperatorNamespace is the default namespace for serving operator e2e tests
-	ServingOperatorNamespace = getenv("TEST_NAMESPACE", "operator-tests")
+	ServingOperatorNamespace = getenv("TEST_NAMESPACE", "knative-serving")
 	// ServingOperatorName is the default operator name for serving operator e2e tests
 	ServingOperatorName = getenv("TEST_RESOURCE", "knative-serving")
 )

test/resources/knativeserving.go

@@ -129,3 +129,56 @@ func getTestKSOperatorCRSpec() v1alpha1.KnativeServingSpec {
 		},
 	}
 }
+
+// WaitForKnativeServingDeploymentState polls the status of the Knative deployments every `interval`
+// until `inState` returns `true` indicating the deployments match the desired deployments.
+func WaitForKnativeServingDeploymentState(clients *test.Clients, namespace string, expectedDeployments []string,
+	inState func(deps *v1.DeploymentList, expectedDeployments []string, err error) (bool, error)) (*v1alpha1.KnativeServing, error) {
+	span := logging.GetEmitableSpan(context.Background(), fmt.Sprintf("WaitForKnativeDeploymentState/%s/%s", expectedDeployments, "KnativeDeploymentIsReady"))
+	defer span.End()
+
+	var lastState *v1alpha1.KnativeServing
+	waitErr := wait.PollImmediate(Interval, Timeout, func() (bool, error) {
+		dpList, err := clients.KubeClient.Kube.AppsV1().Deployments(namespace).List(metav1.ListOptions{})
+		return inState(dpList, expectedDeployments, err)
+	})
+
+	if waitErr != nil {
+		return lastState, waitErr
+	}
+	return lastState, nil
+}
+
+
+// IsKnativeServingDeploymentReady will check the status conditions of the deployments and return true if the deployments meet the desired status.
+func IsKnativeServingDeploymentReady(dpList *v1.DeploymentList, expectedDeployments []string, err error) (bool, error) {
+	if err != nil {
+		return false, err
+	}
+	if len(dpList.Items) != len(expectedDeployments) {
+		errMessage := fmt.Sprintf("The expected number of deployments is %v, and got %v.", len(expectedDeployments), len(dpList.Items))
+		return false, errors.New(errMessage)
+	}
+	for _, deployment := range dpList.Items {
+		if !stringInList(deployment.Name, expectedDeployments) {
+			errMessage := fmt.Sprintf("The deployment %v is not found in the expected list of deployment.", deployment.Name)
+			return false, errors.New(errMessage)
+		}
+		for _, c := range deployment.Status.Conditions {
+			if c.Type == v1.DeploymentAvailable && c.Status != corev1.ConditionTrue {
+				errMessage := fmt.Sprintf("The deployment %v is not ready.", deployment.Name)
+				return false, errors.New(errMessage)
+			}
+		}
+	}
+	return true, nil
+}
+
+func stringInList(a string, list []string) bool {
+	for _, b := range list {
+		if b == a {
+			return true
+		}
+	}
+	return false
+}

test/resources/verify.go

@@ -237,3 +237,11 @@ func verifyNoKSOperatorCR(clients *test.Clients) error {
 	}
 	return nil
 }
+
+// AssertKSOperatorDeploymentStatus verifies if the Knative deployments reach the READY status.
+func AssertKSOperatorDeploymentStatus(t *testing.T, clients *test.Clients, namespace string, expectedDeployments []string) {
+	if _, err := WaitForKnativeServingDeploymentState(clients, namespace, expectedDeployments,
+		IsKnativeServingDeploymentReady); err != nil {
+		t.Fatalf("Knative Serving deployments failed to meet the expected deployments: %v", err)
+	}
+}

test/upgrade/servingoperator_postupgrade_test.go

@@ -18,8 +18,6 @@ package e2e
 import (
 	"testing"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"knative.dev/pkg/test/logstream"
 	"knative.dev/serving-operator/test"
 	"knative.dev/serving-operator/test/client"
@@ -38,60 +36,17 @@ func TestKnativeServingPostUpgrade(t *testing.T) {
 		Namespace:      test.ServingOperatorNamespace,
 	}
 
-	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
-	defer test.TearDown(clients, names)
-
 	// Create a KnativeServing custom resource, if it does not exist
 	if _, err := resources.EnsureKnativeServingExists(clients.KnativeServing(), names); err != nil {
 		t.Fatalf("KnativeService %q failed to create: %v", names.KnativeServing, err)
 	}
 
-	// Test if KnativeServing can reach the READY status after upgrade
-	t.Run("create", func(t *testing.T) {
-		resources.AssertKSOperatorCRReadyStatus(t, clients, names)
-	})
-
 	// Verify if resources match the latest requirement after upgrade
 	t.Run("verify resources", func(t *testing.T) {
-		resources.AssertKSOperatorCRReadyStatus(t, clients, names)
 		// TODO: We only verify the deployment, but we need to add other resources as well, like ServiceAccount, ClusterRoleBinding, etc.
 		expectedDeployments := []string{"networking-istio", "webhook", "controller", "activator", "autoscaler-hpa",
 			"autoscaler"}
-		ksVerifyDeployment(t, clients, names, expectedDeployments)
-	})
-
-	// TODO: We will add one or sections here to run the tests tagged with postupgrade in knative serving.
-
-	// Delete the KnativeServing to see if all resources will be removed after upgrade
-	t.Run("delete", func(t *testing.T) {
+		resources.AssertKSOperatorDeploymentStatus(t, clients, names.Namespace, expectedDeployments)
 		resources.AssertKSOperatorCRReadyStatus(t, clients, names)
-		resources.KSOperatorCRDelete(t, clients, names)
 	})
 }
-
-// ksVerifyDeployment verify whether the deployments have the correct number and names.
-func ksVerifyDeployment(t *testing.T, clients *test.Clients, names test.ResourceNames,
-	expectedDeployments []string) {
-	dpList, err := clients.KubeClient.Kube.AppsV1().Deployments(names.Namespace).List(metav1.ListOptions{})
-	assertEqual(t, err, nil)
-	assertEqual(t, len(dpList.Items), len(expectedDeployments))
-	for _, deployment := range dpList.Items {
-		assertEqual(t, stringInList(deployment.Name, expectedDeployments), true)
-	}
-}
-
-func assertEqual(t *testing.T, actual, expected interface{}) {
-	if actual == expected {
-		return
-	}
-	t.Fatalf("expected does not equal actual. \nExpected: %v\nActual: %v", expected, actual)
-}
-
-func stringInList(a string, list []string) bool {
-	for _, b := range list {
-		if b == a {
-			return true
-		}
-	}
-	return false
-}