Skip to content

Commit

Permalink
[Cherry-pick for v1.8.2] (#94)
Browse files Browse the repository at this point in the history
* chore(log-mgmt): Refactoring code for better log/error mgmt (#91)

Signed-off-by: shubhamchaudhary <shubham.chaudhary@mayadata.io>

* chore(resourceRequirements): Adding resource requirements in chaos pod (#93)

Signed-off-by: shubhamchaudhary <shubham.chaudhary@mayadata.io>

Co-authored-by: Shubham Chaudhary <shubham.chaudhary@mayadata.io>
  • Loading branch information
Karthik Satchitanand and ispeakc0de authored Sep 29, 2020
1 parent d1f71e3 commit 51c72b7
Show file tree
Hide file tree
Showing 56 changed files with 3,140 additions and 162 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
build/_output/
.idea/
vendor/modules.txt
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ test:
@echo "------------------"
@echo "Aquire YAML for performing tests"
@echo "------------------"
@mkdir -p ./build/_output/test;wget -q -N https://raw.githubusercontent.com/litmuschaos/chaos-operator/master/deploy/chaos_crds.yaml -P ./build/_output/test ;wget -q -N https://raw.githubusercontent.com/litmuschaos/chaos-operator/master/deploy/rbac.yaml -P ./build/_output/test; wget -q -N https://raw.githubusercontent.com/litmuschaos/chaos-operator/master/deploy/operator.yaml -P ./build/_output/test
@mkdir -p ./build/_output/test;wget -q -N https://raw.githubusercontent.com/litmuschaos/chaos-operator/master/deploy/chaos_crds.yaml -P ./build/_output/test ;wget -q -N https://raw.githubusercontent.com/litmuschaos/chaos-operator/master/deploy/rbac.yaml -P ./build/_output/test;wget -q -N https://raw.githubusercontent.com/litmuschaos/chaos-operator/master/tests/manifest/pod_delete_rbac.yaml -P ./build/_output/test;wget -q -N https://raw.githubusercontent.com/litmuschaos/chaos-operator/master/deploy/operator.yaml -P ./build/_output/test
@echo "------------------"
@echo "--> Run Go Test"
@echo "------------------"
Expand Down
59 changes: 42 additions & 17 deletions bin/runner.go
Original file line number Diff line number Diff line change
@@ -1,32 +1,49 @@
package main

import (
"k8s.io/klog"

"github.com/litmuschaos/chaos-runner/pkg/log"
"github.com/litmuschaos/chaos-runner/pkg/utils"
"github.com/litmuschaos/chaos-runner/pkg/utils/analytics"
"github.com/sirupsen/logrus"
)

func init() {
// Log as JSON instead of the default ASCII formatter.
logrus.SetFormatter(&logrus.TextFormatter{
FullTimestamp: true,
DisableSorting: true,
DisableLevelTruncation: true,
})
}

func main() {

engineDetails := utils.EngineDetails{}
clients := utils.ClientSets{}
// Getting kubeConfig and Generate ClientSets
if err := clients.GenerateClientSetFromKubeConfig(); err != nil {
klog.Errorf("Unable to create ClientSets, error: %v", err)
return
log.Fatalf("Unable to create ClientSets, error: %v", err)
}
// Fetching all the ENV's needed
utils.GetOsEnv(&engineDetails)
klog.V(0).Infoln("Experiments List: ", engineDetails.Experiments, " ", "Engine Name: ", engineDetails.Name, " ", "appLabels : ", engineDetails.AppLabel, " ", "appKind: ", engineDetails.AppKind, " ", "Service Account Name: ", engineDetails.SvcAccount, "Engine Namespace: ", engineDetails.EngineNamespace)
log.InfoWithValues("Experiments details are as follows", logrus.Fields{
"Experiments List": engineDetails.Experiments,
"Engine Name": engineDetails.Name,
"appLabels": engineDetails.AppLabel,
"appKind": engineDetails.AppKind,
"Service Account Name": engineDetails.SvcAccount,
"Engine Namespace": engineDetails.EngineNamespace,
})

experimentList := utils.CreateExperimentList(&engineDetails)
if err := utils.InitialPatchEngine(engineDetails, clients, experimentList); err != nil {
klog.Errorf("Unable to create Initial ExpeirmentStatus in ChaosEngine, due to error: %v", err)
log.Errorf("Unable to patch Initial ExperimentStatus in ChaosEngine, error: %v", err)
}
recorder, err := utils.NewEventRecorder(clients, engineDetails)
if err != nil {
klog.Errorf("Unable to initiate EventRecorder for Chaos-Runner, would not be able to add events")
log.Errorf("Unable to initiate EventRecorder for Chaos-Runner, would not be able to add events, error: %v", err)
}

// Steps for each Experiment
for _, experiment := range experimentList {

Expand All @@ -35,51 +52,59 @@ func main() {
analytics.TriggerAnalytics(experiment.Name, engineDetails.ClientUUID)
}

// derive the required field from the experiment & engine and set into experimentDetails struct
if err := experiment.SetValueFromChaosResources(&engineDetails, clients); err != nil {
klog.V(0).Infof("Unable to set values from Chaos Resources due to error: %v", err)
log.Errorf("Unable to set values from Chaos Resources, error: %v", err)
recorder.ExperimentSkipped(experiment.Name, utils.ExperimentNotFoundErrorReason)
continue
}

// derive the envs from the chaos experiment and override their values from chaosengine if any
if err := experiment.SetENV(engineDetails, clients); err != nil {
klog.V(0).Infof("Unable to patch ENV due to error: %v", err)
log.Errorf("Unable to patch ENV, error: %v", err)
recorder.ExperimentSkipped(experiment.Name, utils.ExperimentEnvParseErrorReason)
continue
}

klog.V(0).Infof("Preparing to run Chaos Experiment: %v", experiment.Name)
log.Infof("Preparing to run Chaos Experiment: %v", experiment.Name)

if err := experiment.PatchResources(engineDetails, clients); err != nil {
klog.V(0).Infof("Unable to patch Chaos Resources required for Chaos Experiment: %v, due to error: %v", experiment.Name, err)

log.Errorf("Unable to patch Chaos Resources required for Chaos Experiment: %v, error: %v", experiment.Name, err)
recorder.ExperimentSkipped(experiment.Name, utils.ExperimentDependencyCheckReason)
continue
}
// generating experiment dependency check event inside chaosengine
recorder.ExperimentDepedencyCheck(experiment.Name)

// Creation of PodTemplateSpec, and Final Job
if err := utils.BuildingAndLaunchJob(&experiment, clients); err != nil {
klog.V(0).Infof("Unable to construct chaos experiment job due to: %v", err)
log.Errorf("Unable to construct chaos experiment job, error: %v", err)
recorder.ExperimentSkipped(experiment.Name, utils.ExperimentJobCreationErrorReason)
continue
}
recorder.ExperimentJobCreate(experiment.Name, experiment.JobName)

klog.V(0).Infof("Started Chaos Experiment Name: %v, with Job Name: %v", experiment.Name, experiment.JobName)
log.Infof("Started Chaos Experiment Name: %v, with Job Name: %v", experiment.Name, experiment.JobName)
// Watching the chaos container till Completion
if err := engineDetails.WatchChaosContainerForCompletion(&experiment, clients); err != nil {
klog.V(0).Infof("Unable to Watch the chaos container, error: %v", err)
log.Errorf("Unable to Watch the chaos container, error: %v", err)
recorder.ExperimentSkipped(experiment.Name, utils.ExperimentChaosContainerWatchErrorReason)
continue
}

log.Infof("Chaos Pod Completed, Experiment Name: %v, with Job Name: %v", experiment.Name, experiment.JobName)

// Will Update the chaosEngine Status
if err := engineDetails.UpdateEngineWithResult(&experiment, clients); err != nil {
klog.V(0).Infof("Unable to Update ChaosEngine Status due to: %v", err)
log.Errorf("Unable to Update ChaosEngine Status, error: %v", err)
}

log.Infof("Chaos Engine has been updated with result, Experiment Name: %v", experiment.Name)

// Delete / retain the Job, using the jobCleanUpPolicy
jobCleanUpPolicy, err := engineDetails.DeleteJobAccordingToJobCleanUpPolicy(&experiment, clients)
if err != nil {
klog.V(0).Infof("Unable to Delete ChaosExperiment Job due to: %v", err)
log.Errorf("Unable to Delete ChaosExperiment Job, error: %v", err)
}
recorder.ExperimentJobCleanUp(&experiment, jobCleanUpPolicy)
}
Expand Down
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@ go 1.13
require (
github.com/imdario/mergo v0.3.9 // indirect
github.com/jpillora/go-ogle-analytics v0.0.0-20161213085824-14b04e0594ef
github.com/litmuschaos/chaos-operator v0.0.0-20200917035646-2633098da932
github.com/litmuschaos/elves v0.0.0-20200704104545-672722932140
github.com/litmuschaos/chaos-operator v0.0.0-20200929102701-8805f49fa9cf
github.com/litmuschaos/elves v0.0.0-20200929113647-fbf07cc3600f
github.com/onsi/ginkgo v1.11.0
github.com/onsi/gomega v1.8.1
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.4.2
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
k8s.io/api v0.17.3
Expand Down
10 changes: 5 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,7 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
Expand All @@ -415,12 +416,11 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
github.com/litmuschaos/chaos-operator v0.0.0-20200917035646-2633098da932 h1:hYL8VZ+40N3TKBPiEwFxbfhzzTLXU1Qvr6yc2u3lA60=
github.com/litmuschaos/chaos-operator v0.0.0-20200917035646-2633098da932/go.mod h1:QHmeCgV1yDMBnNk5tz0c9OVGUUC7MWrVIAlrPWMRhK4=
github.com/litmuschaos/elves v0.0.0-20200502084504-13be0e6937ee h1:k0ry/r6fbUXp48eGioI9SbVfbSQPBJdhNbkHg7RpHB0=
github.com/litmuschaos/chaos-operator v0.0.0-20200929102701-8805f49fa9cf h1:itW05EDZv/G8cNFiXmXTryWT6fezwVBTWYiiDH0u4j0=
github.com/litmuschaos/chaos-operator v0.0.0-20200929102701-8805f49fa9cf/go.mod h1:QHmeCgV1yDMBnNk5tz0c9OVGUUC7MWrVIAlrPWMRhK4=
github.com/litmuschaos/elves v0.0.0-20200502084504-13be0e6937ee/go.mod h1:DsbHGNUq/78NZozWVVI9Q6eBei4I+JjlkkD5aibJ3MQ=
github.com/litmuschaos/elves v0.0.0-20200704104545-672722932140 h1:H6xJtfYX++HR0FB58KDs73PTU1Nt2UiOj3glmv9uHy0=
github.com/litmuschaos/elves v0.0.0-20200704104545-672722932140/go.mod h1:DsbHGNUq/78NZozWVVI9Q6eBei4I+JjlkkD5aibJ3MQ=
github.com/litmuschaos/elves v0.0.0-20200929113647-fbf07cc3600f h1:t/DDHFV9aVDCwzW13+1evM6DiycUmGqNh+H76NvXTlA=
github.com/litmuschaos/elves v0.0.0-20200929113647-fbf07cc3600f/go.mod h1:DsbHGNUq/78NZozWVVI9Q6eBei4I+JjlkkD5aibJ3MQ=
github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA=
github.com/lucas-clemente/aes12 v0.0.0-20171027163421-cd47fb39b79f/go.mod h1:JpH9J1c9oX6otFSgdUHwUBUizmKlrMjxWnIAjff4m04=
github.com/lucas-clemente/quic-clients v0.1.0/go.mod h1:y5xVIEoObKqULIKivu+gD/LU90pL73bTdtQjPBvtCBk=
Expand Down
61 changes: 61 additions & 0 deletions pkg/log/log.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package log

import (
logrus "github.com/sirupsen/logrus"
)

//Fatalf Logs first and then calls `logger.Exit(1)`
// logging level is set to Panic.
func Fatalf(msg string, err error) {
logrus.WithFields(logrus.Fields{}).Fatalf(msg, err)
}

//Fatal Logs first and then calls `logger.Exit(1)`
// logging level is set to Panic.
func Fatal(msg string) {
logrus.WithFields(logrus.Fields{}).Fatal(msg)
}

//Infof log the General operational entries about what's going on inside the application
func Infof(msg string, val ...interface{}) {
logrus.WithFields(logrus.Fields{}).Infof(msg, val...)
}

//Info log the General operational entries about what's going on inside the application
func Info(msg string) {
logrus.WithFields(logrus.Fields{}).Infof(msg)
}

// InfoWithValues log the General operational entries about what's going on inside the application
// It also print the extra key values pairs
func InfoWithValues(msg string, val map[string]interface{}) {
logrus.WithFields(val).Info(msg)
}

// ErrorWithValues log the Error entries happening inside the code
// It also print the extra key values pairs
func ErrorWithValues(msg string, val map[string]interface{}) {
logrus.WithFields(val).Error(msg)
}

//Warn log the Non-critical entries that deserve eyes.
func Warn(msg string) {
logrus.WithFields(logrus.Fields{}).Warn(msg)
}

//Warnf log the Non-critical entries that deserve eyes.
func Warnf(msg string, val ...interface{}) {
logrus.WithFields(logrus.Fields{}).Warnf(msg, val...)
}

//Errorf used for errors that should definitely be noted.
// Commonly used for hooks to send errors to an error tracking service.
func Errorf(msg string, err ...interface{}) {
logrus.WithFields(logrus.Fields{}).Errorf(msg, err...)
}

//Error used for errors that should definitely be noted.
// Commonly used for hooks to send errors to an error tracking service
func Error(msg string) {
logrus.WithFields(logrus.Fields{}).Error(msg)
}
6 changes: 3 additions & 3 deletions pkg/utils/analytics/analytics.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package analytics

import (
ga "github.com/jpillora/go-ogle-analytics"
"k8s.io/klog"
"github.com/litmuschaos/chaos-runner/pkg/log"
)

const (
Expand All @@ -24,11 +24,11 @@ const (
func TriggerAnalytics(experimentName string, uuid string) {
client, err := ga.NewClient(clientID)
if err != nil {
klog.Error(err, "GA Client ID Error")
log.Errorf("Unable to create GA client, error: %v", err)
}
client.ClientID(uuid)
err = client.Send(ga.NewEvent(category, action).Label(experimentName))
if err != nil {
klog.Infoln("Unable to send GA event", err)
log.Errorf("Unable to send GA event, error: %v", err)
}
}
14 changes: 10 additions & 4 deletions pkg/utils/builders.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,12 @@ func buildContainerSpec(experiment *ExperimentDetails, envVar []corev1.EnvVar) (

}

if !reflect.DeepEqual(experiment.ResourceRequirements, corev1.ResourceRequirements{}) {

containerSpec.WithResourceRequirements(experiment.ResourceRequirements)

}

if experiment.VolumeOpts.VolumeMounts != nil {
containerSpec.WithVolumeMountsNew(experiment.VolumeOpts.VolumeMounts)
}
Expand Down Expand Up @@ -83,13 +89,13 @@ func BuildingAndLaunchJob(experiment *ExperimentDetails, clients ClientSets) err
//Build Container to add in the Pod
containerForPod, err := buildContainerSpec(experiment, envVar)
if err != nil {
return errors.Wrapf(err, "Unable to build Container for Chaos Experiment, due to error: %v", err)
return errors.Errorf("Unable to build Container for Chaos Experiment, error: %v", err)
}
// Will build a PodSpecTemplate
pod, err := buildPodTemplateSpec(experiment, containerForPod)
if err != nil {

return errors.Wrapf(err, "Unable to build PodTemplateSpec for Chaos Experiment, due to error: %v", err)
return errors.Errorf("Unable to build PodTemplateSpec for Chaos Experiment, error: %v", err)
}
// Build JobSpec Template
jobspec, err := buildJobSpec(pod)
Expand All @@ -99,11 +105,11 @@ func BuildingAndLaunchJob(experiment *ExperimentDetails, clients ClientSets) err
//Build Job
job, err := experiment.buildJob(pod, jobspec)
if err != nil {
return errors.Wrapf(err, "Unable to Build ChaosExperiment Job, due to error: %v", err)
return errors.Errorf("Unable to Build ChaosExperiment Job, error: %v", err)
}
// Creating the Job
if err = experiment.launchJob(job, clients); err != nil {
return errors.Wrapf(err, "Unable to launch ChaosExperiment Job, due to error: %v", err)
return errors.Errorf("Unable to launch ChaosExperiment Job, error: %v", err)
}
return nil
}
Expand Down
24 changes: 12 additions & 12 deletions pkg/utils/configMapUtils.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,23 @@ package utils

import (
"github.com/litmuschaos/chaos-operator/pkg/apis/litmuschaos/v1alpha1"
"github.com/litmuschaos/chaos-runner/pkg/log"
"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog"
)

//PatchConfigMaps patches configmaps in experimentDetails struct.
func (expDetails *ExperimentDetails) PatchConfigMaps(clients ClientSets, engineDetails EngineDetails) error {
err := expDetails.SetConfigMaps(clients, engineDetails)
if err != nil {
if err := expDetails.SetConfigMaps(clients, engineDetails); err != nil {
return err
}

klog.V(0).Infof("Validating configmaps specified in the ChaosExperiment & ChaosEngine")
err = expDetails.ValidateConfigMaps(clients)
if err != nil {
log.Info("Validating configmaps specified in the ChaosExperiment & ChaosEngine")

if err := expDetails.ValidateConfigMaps(clients); err != nil {
return err
}

return nil
}

Expand Down Expand Up @@ -55,21 +55,21 @@ func (expDetails *ExperimentDetails) ValidateConfigMaps(clients ClientSets) erro

for _, v := range expDetails.ConfigMaps {
if v.Name == "" || v.MountPath == "" {
return errors.New("Incomplete Information in ConfigMap, will skip execution")
return errors.Errorf("Incomplete Information in ConfigMap, will skip execution")
}
err := clients.ValidateConfigMap(v.Name, expDetails)
if err != nil {
return errors.Wrapf(err, "Unable to get ConfigMap with Name: %v, in namespace: %v", v.Name, expDetails.Namespace)
return errors.Errorf("Unable to get ConfigMap with Name: %v, in namespace: %v", v.Name, expDetails.Namespace)
}
klog.V(0).Infof("Successfully Validated ConfigMap: %v", v.Name)
log.Infof("Successfully Validated ConfigMap: %v", v.Name)
}
return nil
}

func getExperimentConfigmaps(clients ClientSets, expDetails *ExperimentDetails) ([]v1alpha1.ConfigMap, error) {
chaosExperimentObj, err := clients.LitmusClient.LitmuschaosV1alpha1().ChaosExperiments(expDetails.Namespace).Get(expDetails.Name, metav1.GetOptions{})
if err != nil {
return nil, errors.Wrapf(err, "Unable to get ChaosExperiment Resource, error: %v", err)
return nil, errors.Errorf("Unable to get ChaosExperiment Resource, error: %v", err)
}
experimentConfigMaps := chaosExperimentObj.Spec.Definition.ConfigMaps

Expand All @@ -80,7 +80,7 @@ func getEngineConfigmaps(clients ClientSets, engineDetails EngineDetails, expDet

chaosEngineObj, err := engineDetails.GetChaosEngine(clients)
if err != nil {
return nil, errors.Wrapf(err, "Unable to get ChaosEngine Resource, error: %v", err)
return nil, errors.Errorf("Unable to get ChaosEngine Resource, error: %v", err)
}
experimentsList := chaosEngineObj.Spec.Experiments
for i := range experimentsList {
Expand All @@ -89,7 +89,7 @@ func getEngineConfigmaps(clients ClientSets, engineDetails EngineDetails, expDet
return engineConfigMaps, nil
}
}
return nil, errors.Wrapf(err, "No experiment found with %v name in ChaosEngine", expDetails.Name)
return nil, errors.Errorf("No experiment found with %v name in ChaosEngine", expDetails.Name)
}

// OverridingConfigMaps will override configmaps from ChaosEngine
Expand Down
2 changes: 1 addition & 1 deletion pkg/utils/expNotFoundPatch.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ func (engineDetails EngineDetails) ExperimentNotFoundPatchEngine(experiment *Exp
var expStatus ExperimentStatus
expStatus.NotFoundExperimentStatus(experiment.Name, engineDetails.Name)
if err := expStatus.PatchChaosEngineStatus(engineDetails, clients); err != nil {
return errors.Wrapf(err, "Unable to Patch ChaosEngine with Status, error: %v", err)
return errors.Errorf("Unable to Patch ChaosEngine with Status, error: %v", err)
}
return nil
}
Loading

0 comments on commit 51c72b7

Please sign in to comment.