commands/.../scorecard: optionally use only a CSV manifest when deployed with OLM

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 - New flags for [`operator-sdk new --type=helm`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#new), which can be used to populate the project with an existing chart. ([#949](https://github.com/operator-framework/operator-sdk/pull/949))
 - Command [`operator-sdk olm-catalog`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#olm-catalog) flag `--update-crds` optionally copies CRD's from `deploy/crds` when creating a new CSV or updating an existing CSV, and `--from-version` uses another versioned CSV manifest as a base for a new CSV version. ([#1016](https://github.com/operator-framework/operator-sdk/pull/1016))
+- New flag `--olm-deployed` to direct the [`scorecard`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#scorecard) command to only use the CSV at `--csv-path` for manifest data, except for those provided to `--cr-manifest`. ([#1044](https://github.com/operator-framework/operator-sdk/pull/1044))
 
 ### Changed
 

commands/operator-sdk/cmd/scorecard.go

@@ -18,21 +18,21 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/operator-framework/operator-sdk/commands/operator-sdk/cmd/scorecard"
 	"github.com/operator-framework/operator-sdk/pkg/scaffold"
+	"github.com/operator-framework/operator-sdk/version"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-
-	"github.com/operator-framework/operator-sdk/commands/operator-sdk/cmd/scorecard"
-	"github.com/operator-framework/operator-sdk/version"
 )
 
 // scorecardConfig stores all scorecard config passed as flags
 type scorecardConfig struct {
 	namespace          string
 	kubeconfigPath     string
 	initTimeout        int
+	olmDeployed        bool
 	csvPath            string
 	basicTests         bool
 	olmTests           bool
@@ -61,6 +61,7 @@ func NewScorecardCmd() *cobra.Command {
 	scorecardCmd.Flags().StringVar(&scConf.namespace, scorecard.NamespaceOpt, "", "Namespace of custom resource created in cluster")
 	scorecardCmd.Flags().StringVar(&scConf.kubeconfigPath, scorecard.KubeconfigOpt, "", "Path to kubeconfig of custom resource created in cluster")
 	scorecardCmd.Flags().IntVar(&scConf.initTimeout, scorecard.InitTimeoutOpt, 10, "Timeout for status block on CR to be created in seconds")
+	scorecardCmd.Flags().BoolVar(&scConf.olmDeployed, scorecard.OlmDeployedOpt, false, "The OLM has deployed the operator. Use only the CSV for test data")
 	scorecardCmd.Flags().StringVar(&scConf.csvPath, scorecard.CSVPathOpt, "", "Path to CSV being tested")
 	scorecardCmd.Flags().BoolVar(&scConf.basicTests, scorecard.BasicTestsOpt, true, "Enable basic operator checks")
 	scorecardCmd.Flags().BoolVar(&scConf.olmTests, scorecard.OLMTestsOpt, true, "Enable OLM integration checks")

commands/operator-sdk/cmd/scorecard/basic_tests.go

@@ -35,7 +35,7 @@ import (
 // status blocks exist or return an error after the timeout.
 func checkSpecAndStat(runtimeClient client.Client, obj *unstructured.Unstructured, noStore bool) error {
 	testSpec := scorecardTest{testType: basicOperator, name: "Spec Block Exists", maximumPoints: 1}
-	testStat := scorecardTest{testType: basicOperator, name: "Status Block Exist", maximumPoints: 1}
+	testStat := scorecardTest{testType: basicOperator, name: "Status Block Exists", maximumPoints: 1}
 	err := wait.Poll(time.Second*1, time.Second*time.Duration(viper.GetInt64(InitTimeoutOpt)), func() (bool, error) {
 		err := runtimeClient.Get(context.TODO(), types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}, obj)
 		if err != nil {

commands/operator-sdk/cmd/scorecard/resource_handler.go

@@ -130,34 +130,9 @@ func createFromYAMLFile(yamlPath string) error {
 		}
 		addResourceCleanup(obj, types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()})
 		if obj.GetKind() == "Deployment" {
-			dep := &appsv1.Deployment{}
-			err = runtimeClient.Get(context.TODO(), types.NamespacedName{Namespace: viper.GetString(NamespaceOpt), Name: deploymentName}, dep)
+			proxyPod, err = getPodFromDeployment(deploymentName, viper.GetString(NamespaceOpt))
 			if err != nil {
-				return fmt.Errorf("failed to get newly created deployment: %v", err)
-			}
-			set := labels.Set(dep.Spec.Selector.MatchLabels)
-			// in some cases, the pod from the old deployment will be picked up instead of the new one
-			err = wait.PollImmediate(time.Second*1, time.Second*60, func() (bool, error) {
-				pods := &v1.PodList{}
-				err = runtimeClient.List(context.TODO(), &client.ListOptions{LabelSelector: set.AsSelector()}, pods)
-				if err != nil {
-					return false, fmt.Errorf("failed to get list of pods in deployment: %v", err)
-				}
-				// make sure the pods exist
-				// there should only be 1 pod per deployment
-				if len(pods.Items) == 1 {
-					// if the pod has a deletion timestamp, it is the old pod; wait for pod with no deletion timestamp
-					if pods.Items[0].GetDeletionTimestamp() == nil {
-						proxyPod = &pods.Items[0]
-						return true, nil
-					}
-				} else {
-					log.Debug("Operator deployment has more than 1 pod")
-				}
-				return false, nil
-			})
-			if err != nil {
-				return fmt.Errorf("failed to get proxyPod: %s", err)
+				return err
 			}
 		}
 	}
@@ -168,6 +143,41 @@ func createFromYAMLFile(yamlPath string) error {
 	return nil
 }
 
+// getPodFromDeployment returns a deployment depName's pod in namespace.
+func getPodFromDeployment(depName, namespace string) (pod *v1.Pod, err error) {
+	dep := &appsv1.Deployment{}
+	err = runtimeClient.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: depName}, dep)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get newly created deployment: %v", err)
+	}
+	set := labels.Set(dep.Spec.Selector.MatchLabels)
+	// In some cases, the pod from the old deployment will be picked up
+	// instead of the new one.
+	err = wait.PollImmediate(time.Second*1, time.Second*60, func() (bool, error) {
+		pods := &v1.PodList{}
+		err = runtimeClient.List(context.TODO(), &client.ListOptions{LabelSelector: set.AsSelector()}, pods)
+		if err != nil {
+			return false, fmt.Errorf("failed to get list of pods in deployment: %v", err)
+		}
+		// Make sure the pods exist. There should only be 1 pod per deployment.
+		if len(pods.Items) == 1 {
+			// If the pod has a deletion timestamp, it is the old pod; wait for
+			// pod with no deletion timestamp
+			if pods.Items[0].GetDeletionTimestamp() == nil {
+				pod = &pods.Items[0]
+				return true, nil
+			}
+		} else {
+			log.Debug("Operator deployment has more than 1 pod")
+		}
+		return false, nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get proxyPod: %s", err)
+	}
+	return pod, nil
+}
+
 // createKubeconfigSecret creates the secret that will be mounted in the operator's container and contains
 // the kubeconfig for communicating with the proxy
 func createKubeconfigSecret() error {
@@ -249,7 +259,7 @@ func addProxyContainer(dep *appsv1.Deployment) {
 		pullPolicy = v1.PullAlways
 	}
 	dep.Spec.Template.Spec.Containers = append(dep.Spec.Template.Spec.Containers, v1.Container{
-		Name:            "scorecard-proxy",
+		Name:            scorecardContainerName,
 		Image:           viper.GetString(ProxyImageOpt),
 		ImagePullPolicy: pullPolicy,
 		Command:         []string{"scorecard-proxy"},
@@ -340,7 +350,8 @@ func getProxyLogs() (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("failed to create kubeclient: %v", err)
 	}
-	req := kubeclient.CoreV1().Pods(proxyPod.GetNamespace()).GetLogs(proxyPod.GetName(), &v1.PodLogOptions{Container: "scorecard-proxy"})
+	logOpts := &v1.PodLogOptions{Container: scorecardContainerName}
+	req := kubeclient.CoreV1().Pods(proxyPod.GetNamespace()).GetLogs(proxyPod.GetName(), logOpts)
 	readCloser, err := req.Stream()
 	if err != nil {
 		return "", fmt.Errorf("failed to get logs: %v", err)