Skip to content

Commit dc64179

Browse files
zhangzujianzhangzujian
committed
use httpGet as liveness/readiness probe method (#4945)
Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>
1 parent 97dc121 commit dc64179

File tree

12 files changed

+107
-127
lines changed

12 files changed

+107
-127
lines changed

‎Makefile

+2
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,7 @@ kind-install-chart: kind-load-image kind-untaint-control-plane
515515
--set global.images.kubeovn.tag=$(VERSION) \
516516
--set replicaCount=$$(echo $$ips | awk -F ',' '{print NF}') \
517517
--set MASTER_NODES="$$(echo $$ips | sed 's/,/\\,/g')" \
518+
--set func.SECURE_SERVING=$(shell echo $${SECURE_SERVING:-false}) \
518519
--set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)
519520

520521
.PHONY: kind-install-chart-ssl
@@ -528,6 +529,7 @@ kind-upgrade-chart: kind-load-image
528529
--set global.images.kubeovn.tag=$(VERSION) \
529530
--set replicaCount=$$(echo $(OVN_DB_IPS) | awk -F ',' '{print NF}') \
530531
--set MASTER_NODES='$(OVN_DB_IPS)' \
532+
--set func.SECURE_SERVING=$(shell echo $${SECURE_SERVING:-false}) \
531533
--set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)
532534
sleep 90
533535
kubectl -n kube-system wait pod --for=condition=ready -l app=ovs --timeout=60s

‎charts/kube-ovn/templates/controller-deploy.yaml

+8-10
Original file line numberDiff line numberDiff line change
@@ -168,19 +168,17 @@ spec:
168168
- mountPath: /var/run/tls
169169
name: kube-ovn-tls
170170
readinessProbe:
171-
exec:
172-
command:
173-
- /kube-ovn/kube-ovn-healthcheck
174-
- --port=10660
175-
- --tls={{- .Values.func.SECURE_SERVING }}
171+
httpGet:
172+
port: 10660
173+
path: /readyz
174+
scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}'
176175
periodSeconds: 3
177176
timeoutSeconds: 5
178177
livenessProbe:
179-
exec:
180-
command:
181-
- /kube-ovn/kube-ovn-healthcheck
182-
- --port=10660
183-
- --tls={{- .Values.func.SECURE_SERVING }}
178+
httpGet:
179+
port: 10660
180+
path: /livez
181+
scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}'
184182
initialDelaySeconds: 300
185183
periodSeconds: 7
186184
failureThreshold: 5

‎charts/kube-ovn/templates/monitor-deploy.yaml

+8-10
Original file line numberDiff line numberDiff line change
@@ -108,22 +108,20 @@ spec:
108108
initialDelaySeconds: 30
109109
periodSeconds: 7
110110
successThreshold: 1
111-
exec:
112-
command:
113-
- /kube-ovn/kube-ovn-healthcheck
114-
- --port=10661
115-
- --tls={{- .Values.func.SECURE_SERVING }}
111+
httpGet:
112+
port: 10661
113+
path: /livez
114+
scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}'
116115
timeoutSeconds: 5
117116
readinessProbe:
118117
failureThreshold: 3
119118
initialDelaySeconds: 30
120119
periodSeconds: 7
121120
successThreshold: 1
122-
exec:
123-
command:
124-
- /kube-ovn/kube-ovn-healthcheck
125-
- --port=10661
126-
- --tls={{- .Values.func.SECURE_SERVING }}
121+
httpGet:
122+
port: 10661
123+
path: /readyz
124+
scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}'
127125
timeoutSeconds: 5
128126
nodeSelector:
129127
kubernetes.io/os: "linux"

‎charts/kube-ovn/templates/ovncni-ds.yaml

+8-10
Original file line numberDiff line numberDiff line change
@@ -152,22 +152,20 @@ spec:
152152
failureThreshold: 3
153153
periodSeconds: 7
154154
successThreshold: 1
155-
exec:
156-
command:
157-
- /kube-ovn/kube-ovn-healthcheck
158-
- --port=10665
159-
- --tls={{- .Values.func.SECURE_SERVING }}
155+
httpGet:
156+
port: 10665
157+
path: /readyz
158+
scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}'
160159
timeoutSeconds: 5
161160
livenessProbe:
162161
failureThreshold: 3
163162
initialDelaySeconds: 30
164163
periodSeconds: 7
165164
successThreshold: 1
166-
exec:
167-
command:
168-
- /kube-ovn/kube-ovn-healthcheck
169-
- --port=10665
170-
- --tls={{- .Values.func.SECURE_SERVING }}
165+
httpGet:
166+
port: 10665
167+
path: /livez
168+
scheme: '{{ ternary "HTTPS" "HTTP" .Values.func.SECURE_SERVING }}'
171169
timeoutSeconds: 5
172170
resources:
173171
requests:

‎cmd/cmdmain.go

-4
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import (
1313

1414
"github.com/kubeovn/kube-ovn/cmd/controller"
1515
"github.com/kubeovn/kube-ovn/cmd/daemon"
16-
"github.com/kubeovn/kube-ovn/cmd/health_check"
1716
"github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller"
1817
"github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker"
1918
"github.com/kubeovn/kube-ovn/cmd/ovn_monitor"
@@ -28,7 +27,6 @@ const (
2827
CmdMonitor = "kube-ovn-monitor"
2928
CmdPinger = "kube-ovn-pinger"
3029
CmdSpeaker = "kube-ovn-speaker"
31-
CmdHealthCheck = "kube-ovn-healthcheck"
3230
CmdOvnLeaderChecker = "kube-ovn-leader-checker"
3331
CmdOvnICController = "kube-ovn-ic-controller"
3432
)
@@ -93,8 +91,6 @@ func main() {
9391
case CmdSpeaker:
9492
dumpProfile()
9593
speaker.CmdMain()
96-
case CmdHealthCheck:
97-
health_check.CmdMain()
9894
case CmdOvnLeaderChecker:
9995
ovn_leader_checker.CmdMain()
10096
case CmdOvnICController:

‎cmd/health_check/health_check.go

-57
This file was deleted.

‎dist/images/Dockerfile

-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,5 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \
2525
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-monitor && \
2626
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \
2727
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \
28-
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-healthcheck && \
2928
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \
3029
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller

‎dist/images/install.sh

+29-30
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ SECURE_SERVING=${SECURE_SERVING:-false}
4141
OVSDB_CON_TIMEOUT=${OVSDB_CON_TIMEOUT:-3}
4242
OVSDB_INACTIVITY_TIMEOUT=${OVSDB_INACTIVITY_TIMEOUT:-10}
4343

44+
PROBE_HTTP_SCHEME="HTTP"
45+
if [ "$SECURE_SERVING" = "true" ]; then
46+
PROBE_HTTP_SCHEME="HTTPS"
47+
fi
48+
4449
# debug
4550
DEBUG_WRAPPER=${DEBUG_WRAPPER:-}
4651

@@ -4161,19 +4166,17 @@ spec:
41614166
- mountPath: /var/run/tls
41624167
name: kube-ovn-tls
41634168
readinessProbe:
4164-
exec:
4165-
command:
4166-
- /kube-ovn/kube-ovn-healthcheck
4167-
- --port=10660
4168-
- --tls=${SECURE_SERVING}
4169+
httpGet:
4170+
port: 10660
4171+
path: /readyz
4172+
scheme: ${PROBE_HTTP_SCHEME}
41694173
periodSeconds: 3
41704174
timeoutSeconds: 5
41714175
livenessProbe:
4172-
exec:
4173-
command:
4174-
- /kube-ovn/kube-ovn-healthcheck
4175-
- --port=10660
4176-
- --tls=${SECURE_SERVING}
4176+
httpGet:
4177+
port: 10660
4178+
path: /livez
4179+
scheme: ${PROBE_HTTP_SCHEME}
41774180
initialDelaySeconds: 300
41784181
periodSeconds: 7
41794182
failureThreshold: 5
@@ -4344,21 +4347,19 @@ spec:
43444347
initialDelaySeconds: 30
43454348
periodSeconds: 7
43464349
successThreshold: 1
4347-
exec:
4348-
command:
4349-
- /kube-ovn/kube-ovn-healthcheck
4350-
- --port=10665
4351-
- --tls=${SECURE_SERVING}
4350+
httpGet:
4351+
port: 10665
4352+
path: /livez
4353+
scheme: ${PROBE_HTTP_SCHEME}
43524354
timeoutSeconds: 5
43534355
readinessProbe:
43544356
failureThreshold: 3
43554357
periodSeconds: 7
43564358
successThreshold: 1
4357-
exec:
4358-
command:
4359-
- /kube-ovn/kube-ovn-healthcheck
4360-
- --port=10665
4361-
- --tls=${SECURE_SERVING}
4359+
httpGet:
4360+
port: 10665
4361+
path: /readyz
4362+
scheme: ${PROBE_HTTP_SCHEME}
43624363
timeoutSeconds: 5
43634364
resources:
43644365
requests:
@@ -4642,22 +4643,20 @@ spec:
46424643
initialDelaySeconds: 30
46434644
periodSeconds: 7
46444645
successThreshold: 1
4645-
exec:
4646-
command:
4647-
- /kube-ovn/kube-ovn-healthcheck
4648-
- --port=10661
4649-
- --tls=${SECURE_SERVING}
4646+
httpGet:
4647+
port: 10661
4648+
path: /livez
4649+
scheme: ${PROBE_HTTP_SCHEME}
46504650
timeoutSeconds: 5
46514651
readinessProbe:
46524652
failureThreshold: 3
46534653
initialDelaySeconds: 30
46544654
periodSeconds: 7
46554655
successThreshold: 1
4656-
exec:
4657-
command:
4658-
- /kube-ovn/kube-ovn-healthcheck
4659-
- --port=10661
4660-
- --tls=${SECURE_SERVING}
4656+
httpGet:
4657+
port: 10661
4658+
path: /readyz
4659+
scheme: ${PROBE_HTTP_SCHEME}
46614660
timeoutSeconds: 5
46624661
nodeSelector:
46634662
kubernetes.io/os: "linux"

‎go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ require (
1313
github.com/digitalocean/go-openvswitch v0.0.0-20240130171624-c0f7d42efe24
1414
github.com/docker/docker v27.1.1+incompatible
1515
github.com/emicklei/go-restful/v3 v3.12.1
16-
github.com/evanphx/json-patch/v5 v5.9.0
16+
github.com/evanphx/json-patch/v5 v5.9.11
17+
github.com/go-logr/logr v1.4.2
1718
github.com/go-logr/stdr v1.2.2
1819
github.com/google/uuid v1.6.0
1920
github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.1
@@ -77,7 +78,6 @@ require (
7778
github.com/go-kit/kit v0.13.0 // indirect
7879
github.com/go-kit/log v0.2.1 // indirect
7980
github.com/go-logfmt/logfmt v0.6.0 // indirect
80-
github.com/go-logr/logr v1.4.2 // indirect
8181
github.com/go-openapi/jsonpointer v0.21.0 // indirect
8282
github.com/go-openapi/jsonreference v0.21.0 // indirect
8383
github.com/go-openapi/swag v0.23.0 // indirect

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -725,8 +725,8 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni
725725
github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
726726
github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
727727
github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
728-
github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
729-
github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
728+
github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
729+
github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
730730
github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
731731
github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
732732
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=

‎pkg/metrics/server.go

+31-1
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,39 @@ package metrics
33
import (
44
"context"
55
"fmt"
6+
"net/http"
67

8+
"github.com/go-logr/logr"
79
"k8s.io/client-go/rest"
810
"k8s.io/klog/v2"
911
ctrl "sigs.k8s.io/controller-runtime"
1012
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
1113
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
14+
15+
"github.com/kubeovn/kube-ovn/pkg/util"
1216
)
1317

18+
func filterProvider(c *rest.Config, httpClient *http.Client) (server.Filter, error) {
19+
return func(log logr.Logger, handler http.Handler) (http.Handler, error) {
20+
filter, err := filters.WithAuthenticationAndAuthorization(c, httpClient)
21+
if err != nil {
22+
return nil, fmt.Errorf("failed to create filter: %w", err)
23+
}
24+
h, err := filter(log, handler)
25+
if err != nil {
26+
return nil, fmt.Errorf("failed to create handler: %w", err)
27+
}
28+
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
29+
switch req.URL.Path {
30+
case "/healthz", "/livez", "/readyz":
31+
handler.ServeHTTP(w, req)
32+
default:
33+
h.ServeHTTP(w, req)
34+
}
35+
}), nil
36+
}, nil
37+
}
38+
1439
func Run(ctx context.Context, config *rest.Config, addr string, secureServing bool) error {
1540
if config == nil {
1641
config = ctrl.GetConfigOrDie()
@@ -26,7 +51,12 @@ func Run(ctx context.Context, config *rest.Config, addr string, secureServing bo
2651
BindAddress: addr,
2752
}
2853
if secureServing {
29-
options.FilterProvider = filters.WithAuthenticationAndAuthorization
54+
options.FilterProvider = filterProvider
55+
}
56+
options.ExtraHandlers = map[string]http.Handler{
57+
"/healthz": http.HandlerFunc(util.DefaultHealthCheckHandler),
58+
"/livez": http.HandlerFunc(util.DefaultHealthCheckHandler),
59+
"/readyz": http.HandlerFunc(util.DefaultHealthCheckHandler),
3060
}
3161
svr, err := server.NewServer(options, config, client)
3262
if err != nil {

‎pkg/util/health_check.go

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package util
2+
3+
import (
4+
"net/http"
5+
6+
"k8s.io/klog/v2"
7+
)
8+
9+
func DefaultHealthCheckHandler(w http.ResponseWriter, _ *http.Request) {
10+
if _, err := w.Write([]byte("ok")); err != nil {
11+
klog.Error(err)
12+
w.WriteHeader(http.StatusInternalServerError)
13+
return
14+
}
15+
16+
w.WriteHeader(http.StatusOK)
17+
}

0 commit comments

Comments
 (0)