diff --git a/config/core/configmaps/deployment.yaml b/config/core/configmaps/deployment.yaml index b8c70ab39da8..5813cb87fd8b 100644 --- a/config/core/configmaps/deployment.yaml +++ b/config/core/configmaps/deployment.yaml @@ -22,7 +22,7 @@ metadata: app.kubernetes.io/component: controller app.kubernetes.io/version: devel annotations: - knative.dev/example-checksum: "e2f637c6" + knative.dev/example-checksum: "720ddb97" data: # This is the Go import path for the binary that is containerized # and substituted here. @@ -108,3 +108,18 @@ data: # ` # This may be "none" or "prefer-spread-revision-over-nodes" (default) # default-affinity-type: "prefer-spread-revision-over-nodes" + + # runtime-class-name contains the selector for which runtimeClassName + # is selected to put in a revision. + # By default, it is not set by Knative. + # + # Example: + # runtime-class-name: | + # "": + # selector: + # use-default-runc: "yes" + # kata: {} + # gvisor: + # selector: + # use-gvisor: "please" + runtime-class-name: "" diff --git a/pkg/deployment/config.go b/pkg/deployment/config.go index 4bb78d33056d..82d513d4f23e 100644 --- a/pkg/deployment/config.go +++ b/pkg/deployment/config.go @@ -19,12 +19,16 @@ package deployment import ( "errors" "fmt" + "strings" "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/yaml" + cm "knative.dev/pkg/configmap" + "knative.dev/pkg/ptr" ) const ( @@ -106,6 +110,7 @@ func defaultConfig() *Config { RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"), QueueSidecarCPURequest: &QueueSidecarCPURequestDefault, DefaultAffinityType: defaultAffinityTypeValue, + RuntimeClassName: "", } // The following code is needed for ConfigMap testing. // defaultConfig must match the example in deployment.yaml which includes: `queue-sidecar-token-audiences: ""` @@ -116,6 +121,52 @@ func defaultConfig() *Config { return cfg } +func (d Config) PodRuntimeClassName(lbs map[string]string) *string { + cfg := map[string]RuntimeClassNameLabelSelector{} + if err := yaml.Unmarshal([]byte(d.RuntimeClassName), &cfg); err != nil { + return nil + } + runtimeClassName := "" + specificity := -1 + for k, v := range cfg { + if !v.Matches(lbs) || v.specificity() < specificity { + continue + } + if v.specificity() > specificity || strings.Compare(k, runtimeClassName) < 0 { + runtimeClassName = k + specificity = v.specificity() + } + } + if runtimeClassName == "" { + return nil + } + return ptr.String(runtimeClassName) +} + +type RuntimeClassNameLabelSelector struct { + Selector map[string]string `json:"selector,omitempty"` +} + +func (s *RuntimeClassNameLabelSelector) specificity() int { + if s.Selector == nil { + return 0 + } + return len(s.Selector) +} + +func (s *RuntimeClassNameLabelSelector) Matches(labels map[string]string) bool { + if s.Selector == nil { + return true + } + for label, expectedValue := range s.Selector { + value, ok := labels[label] + if !ok || expectedValue != value { + return false + } + } + return true +} + // NewConfigFromMap creates a DeploymentConfig from the supplied Map. func NewConfigFromMap(configMap map[string]string) (*Config, error) { nc := defaultConfig() @@ -147,6 +198,8 @@ func NewConfigFromMap(configMap map[string]string) (*Config, error) { cm.AsStringSet(queueSidecarTokenAudiencesKey, &nc.QueueSidecarTokenAudiences), cm.AsString(queueSidecarRooCAKey, &nc.QueueSidecarRootCA), + + cm.AsString("runtime-class-name", &nc.RuntimeClassName), ); err != nil { return nil, err } @@ -240,4 +293,7 @@ type Config struct { // DefaultAffinityType is a string that controls what affinity rules will be automatically // applied to the PodSpec of all Knative services. DefaultAffinityType AffinityType + + // RuntimeClassName is which runtime the Pod will use + RuntimeClassName string } diff --git a/pkg/deployment/config_test.go b/pkg/deployment/config_test.go index b4965126d5fd..1e1d265e7809 100644 --- a/pkg/deployment/config_test.go +++ b/pkg/deployment/config_test.go @@ -23,10 +23,12 @@ import ( "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "knative.dev/pkg/ptr" "knative.dev/pkg/system" "knative.dev/serving/test/conformance/api/shared" @@ -334,6 +336,63 @@ func TestControllerConfiguration(t *testing.T) { QueueSidecarTokenAudiences: sets.New("foo"), DefaultAffinityType: defaultAffinityTypeValue, }, + }, { + name: "runtime class name defaults to nothing", + wantErr: false, + data: map[string]string{ + QueueSidecarImageKey: defaultSidecarImage, + }, + wantConfig: &Config{ + DigestResolutionTimeout: digestResolutionTimeoutDefault, + ProgressDeadline: ProgressDeadlineDefault, + QueueSidecarCPURequest: &QueueSidecarCPURequestDefault, + QueueSidecarImage: defaultSidecarImage, + QueueSidecarTokenAudiences: sets.New(""), + RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"), + RuntimeClassName: "", + }, + }, { + name: "runtime class name with wildcard", + wantErr: false, + wantConfig: &Config{ + RuntimeClassName: "gvisor: {}", + DigestResolutionTimeout: digestResolutionTimeoutDefault, + ProgressDeadline: ProgressDeadlineDefault, + QueueSidecarCPURequest: &QueueSidecarCPURequestDefault, + QueueSidecarImage: defaultSidecarImage, + QueueSidecarTokenAudiences: sets.New(""), + RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"), + }, + data: map[string]string{ + "runtime-class-name": "gvisor: {}", + QueueSidecarImageKey: defaultSidecarImage, + }, + }, { + name: "runtime class name with wildcard and label selectors", + wantErr: false, + wantConfig: &Config{ + RuntimeClassName: `--- +gvisor: {} +kata: + selector: + some: value-here +`, + DigestResolutionTimeout: digestResolutionTimeoutDefault, + ProgressDeadline: ProgressDeadlineDefault, + QueueSidecarCPURequest: &QueueSidecarCPURequestDefault, + QueueSidecarImage: defaultSidecarImage, + QueueSidecarTokenAudiences: sets.New(""), + RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"), + }, + data: map[string]string{ + "runtime-class-name": `--- +gvisor: {} +kata: + selector: + some: value-here +`, + QueueSidecarImageKey: defaultSidecarImage, + }, }} for _, tt := range configTests { @@ -369,3 +428,79 @@ func quantity(val string) *resource.Quantity { r := resource.MustParse(val) return &r } + +func TestPodRuntimeClassName(t *testing.T) { + ts := []struct { + name string + serviceLabels map[string]string + runtimeClassNames string + want *string + }{{ + name: "empty", + serviceLabels: map[string]string{}, + runtimeClassNames: "", + want: nil, + }, { + name: "wildcard set", + serviceLabels: map[string]string{}, + runtimeClassNames: `gvisor: {}`, + want: ptr.String("gvisor"), + }, { + name: "set via label", + serviceLabels: map[string]string{ + "very-cool": "indeed", + }, + runtimeClassNames: `--- +gvisor: {} +kata: + selector: + very-cool: indeed +`, + want: ptr.String("kata"), + }, { + name: "no default only labels with set labels", + serviceLabels: map[string]string{ + "very-cool": "indeed", + }, + runtimeClassNames: `--- +"": {} +kata: + selector: + very-cool: indeed +`, + want: ptr.String("kata"), + }, { + name: "no default only labels with set no labels", + serviceLabels: map[string]string{}, + runtimeClassNames: `--- +"": {} +kata: + selector: + very-cool: indeed +`, + want: nil, + }, { + name: "invalid configuration", + serviceLabels: map[string]string{}, + runtimeClassNames: `--- +"": "" +`, + want: nil, + }} + + for _, tt := range ts { + tt := tt + t.Run(tt.name, func(t *testing.T) { + if tt.serviceLabels == nil { + tt.serviceLabels = map[string]string{} + } + defaults := defaultConfig() + defaults.RuntimeClassName = tt.runtimeClassNames + got, want := defaults.PodRuntimeClassName(tt.serviceLabels), tt.want + + if !equality.Semantic.DeepEqual(got, want) { + t.Errorf("PodRuntimeClassName() = %v, wanted %v", got, want) + } + }) + } +} diff --git a/pkg/deployment/zz_generated.deepcopy.go b/pkg/deployment/zz_generated.deepcopy.go index f7362289a566..32638310e00d 100644 --- a/pkg/deployment/zz_generated.deepcopy.go +++ b/pkg/deployment/zz_generated.deepcopy.go @@ -84,3 +84,26 @@ func (in *Config) DeepCopy() *Config { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeClassNameLabelSelector) DeepCopyInto(out *RuntimeClassNameLabelSelector) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeClassNameLabelSelector. +func (in *RuntimeClassNameLabelSelector) DeepCopy() *RuntimeClassNameLabelSelector { + if in == nil { + return nil + } + out := new(RuntimeClassNameLabelSelector) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/reconciler/revision/resources/deploy.go b/pkg/reconciler/revision/resources/deploy.go index d17cfe830b54..f62c5514f2d3 100644 --- a/pkg/reconciler/revision/resources/deploy.go +++ b/pkg/reconciler/revision/resources/deploy.go @@ -210,6 +210,9 @@ func makePodSpec(rev *v1.Revision, cfg *config.Config) (*corev1.PodSpec, error) podSpec := BuildPodSpec(rev, append(BuildUserContainers(rev), *queueContainer), cfg) podSpec.Volumes = append(podSpec.Volumes, extraVolumes...) + if val := cfg.Deployment.PodRuntimeClassName(rev.ObjectMeta.Labels); podSpec.RuntimeClassName == nil { + podSpec.RuntimeClassName = val + } if cfg.Observability.EnableVarLogCollection { podSpec.Volumes = append(podSpec.Volumes, varLogVolume) diff --git a/pkg/reconciler/revision/resources/deploy_test.go b/pkg/reconciler/revision/resources/deploy_test.go index bf1038130499..bacd5f8b5d93 100644 --- a/pkg/reconciler/revision/resources/deploy_test.go +++ b/pkg/reconciler/revision/resources/deploy_test.go @@ -407,6 +407,12 @@ func withPrependedVolumeMounts(volumeMounts ...corev1.VolumeMount) containerOpti } } +func withRuntimeClass(name string) podSpecOption { + return func(ps *corev1.PodSpec) { + ps.RuntimeClassName = ptr.String(name) + } +} + func podSpec(containers []corev1.Container, opts ...podSpecOption) *corev1.PodSpec { podSpec := defaultPodSpec.DeepCopy() podSpec.Containers = containers @@ -1526,6 +1532,107 @@ func TestMakePodSpec(t *testing.T) { } }, ), + }, { + name: "with runtime-class-name set", + dc: deployment.Config{ + RuntimeClassName: `--- +gvisor: {} +`, + }, + rev: revision("bar", "foo", + withContainers([]corev1.Container{{ + Name: servingContainerName, + Image: "busybox", + Ports: buildContainerPorts(v1.DefaultUserPort), + ReadinessProbe: withHTTPReadinessProbe(v1.DefaultUserPort), + }}), + ), + want: podSpec([]corev1.Container{ + servingContainer(func(container *corev1.Container) { + container.Image = "busybox" + }), + queueContainer( + withEnvVar("SERVING_READINESS_PROBE", `{"httpGet":{"path":"/","port":8080,"host":"127.0.0.1","scheme":"HTTP"}}`), + ), + }, withRuntimeClass("gvisor")), + }, { + name: "with runtime-class-name set requiring selector and no label set in revision", + dc: deployment.Config{ + RuntimeClassName: `--- +gvisor: + selector: + this-one: specifically +`, + }, + rev: revision("bar", "foo", + withContainers([]corev1.Container{{ + Name: servingContainerName, + Image: "busybox", + Ports: buildContainerPorts(v1.DefaultUserPort), + ReadinessProbe: withHTTPReadinessProbe(v1.DefaultUserPort), + }}), + ), + want: podSpec([]corev1.Container{ + servingContainer(func(container *corev1.Container) { + container.Image = "busybox" + }), + queueContainer( + withEnvVar("SERVING_READINESS_PROBE", `{"httpGet":{"path":"/","port":8080,"host":"127.0.0.1","scheme":"HTTP"}}`), + ), + }), + }, { + name: "with runtime-class-name set requiring selector and label set in revision", + dc: deployment.Config{ + RuntimeClassName: `--- +gvisor: + selector: + this-one: specifically +`, + }, + rev: revision("bar", "foo", + WithRevisionLabel("this-one", "specifically"), + withContainers([]corev1.Container{{ + Name: servingContainerName, + Image: "busybox", + Ports: buildContainerPorts(v1.DefaultUserPort), + ReadinessProbe: withHTTPReadinessProbe(v1.DefaultUserPort), + }}), + ), + want: podSpec([]corev1.Container{ + servingContainer(func(container *corev1.Container) { + container.Image = "busybox" + }), + queueContainer( + withEnvVar("SERVING_READINESS_PROBE", `{"httpGet":{"path":"/","port":8080,"host":"127.0.0.1","scheme":"HTTP"}}`), + ), + }, withRuntimeClass("gvisor")), + }, { + name: "with multiple runtime-class-name set and label selector for one", + dc: deployment.Config{ + RuntimeClassName: `--- +gvisor: {} +kata: + selector: + specific: this-one +`, + }, + rev: revision("bar", "foo", + WithRevisionLabel("specific", "this-one"), + withContainers([]corev1.Container{{ + Name: servingContainerName, + Image: "busybox", + Ports: buildContainerPorts(v1.DefaultUserPort), + ReadinessProbe: withHTTPReadinessProbe(v1.DefaultUserPort), + }}), + ), + want: podSpec([]corev1.Container{ + servingContainer(func(container *corev1.Container) { + container.Image = "busybox" + }), + queueContainer( + withEnvVar("SERVING_READINESS_PROBE", `{"httpGet":{"path":"/","port":8080,"host":"127.0.0.1","scheme":"HTTP"}}`), + ), + }, withRuntimeClass("kata")), }} for _, test := range tests {