Skip to content

Commit bdf152d

Browse files
zhangzujianzhangzujian
committed
fix gateway node check for centralized ecmp subnets
Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>
1 parent 96ad245 commit bdf152d

File tree

11 files changed

+126
-34
lines changed

11 files changed

+126
-34
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ dist/images/test-server
55
dist/images/kube-ovn
66
dist/images/kube-ovn-cmd
77
dist/images/kube-ovn-daemon
8+
dist/images/kube-ovn-controller
89
dist/images/kube-ovn-pinger
910
dist/images/kube-ovn-webhook
1011
dist/windows/kube-ovn.exe

‎Makefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ build-go:
117117
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/kube-ovn -v ./cmd/cni
118118
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-cmd -v ./cmd
119119
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-daemon -v ./cmd/daemon
120-
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-pinger -v ./cmd/pinger
120+
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-controller -v ./cmd/controller
121121
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/test-server -v ./test/server
122122

123123
.PHONY: build-go-windows
@@ -131,7 +131,7 @@ build-go-arm:
131131
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/kube-ovn -v ./cmd/cni
132132
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-cmd -v ./cmd
133133
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-daemon -v ./cmd/daemon
134-
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-pinger -v ./cmd/pinger
134+
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-controller -v ./cmd/controller
135135

136136
.PHONY: build-kube-ovn
137137
build-kube-ovn: build-debug build-go

‎charts/kube-ovn/templates/controller-deploy.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ spec:
147147
capabilities:
148148
add:
149149
- NET_BIND_SERVICE
150+
- NET_RAW
150151
env:
151152
- name: ENABLE_SSL
152153
value: "{{ .Values.networking.ENABLE_SSL }}"

‎cmd/cmdmain.go

-5
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111

1212
"k8s.io/klog/v2"
1313

14-
"github.com/kubeovn/kube-ovn/cmd/controller"
1514
"github.com/kubeovn/kube-ovn/cmd/health_check"
1615
"github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller"
1716
"github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker"
@@ -22,7 +21,6 @@ import (
2221
)
2322

2423
const (
25-
CmdController = "kube-ovn-controller"
2624
CmdMonitor = "kube-ovn-monitor"
2725
CmdSpeaker = "kube-ovn-speaker"
2826
CmdWebhook = "kube-ovn-webhook"
@@ -91,9 +89,6 @@ func dumpProfile() {
9189
func main() {
9290
cmd := filepath.Base(os.Args[0])
9391
switch cmd {
94-
case CmdController:
95-
dumpProfile()
96-
controller.CmdMain()
9792
case CmdMonitor:
9893
dumpProfile()
9994
ovn_monitor.CmdMain()

‎cmd/controller/cmdmain.go

+92
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"os/signal"
7+
"path/filepath"
8+
"runtime/pprof"
9+
"syscall"
10+
"time"
11+
12+
"k8s.io/klog/v2"
13+
14+
"github.com/kubeovn/kube-ovn/cmd/pinger"
15+
"github.com/kubeovn/kube-ovn/pkg/util"
16+
)
17+
18+
const (
19+
CmdController = "kube-ovn-controller"
20+
CmdPinger = "kube-ovn-pinger"
21+
)
22+
23+
const timeFormat = "2006-01-02_15:04:05"
24+
25+
func dumpProfile() {
26+
ch1 := make(chan os.Signal, 1)
27+
ch2 := make(chan os.Signal, 1)
28+
signal.Notify(ch1, syscall.SIGUSR1)
29+
signal.Notify(ch2, syscall.SIGUSR2)
30+
go func() {
31+
for {
32+
<-ch1
33+
name := fmt.Sprintf("cpu-profile-%s.pprof", time.Now().Format(timeFormat))
34+
path := filepath.Join(os.TempDir(), name)
35+
f, err := os.Create(path) // #nosec G303,G304
36+
if err != nil {
37+
klog.Errorf("failed to create cpu profile file: %v", err)
38+
return
39+
}
40+
if err = pprof.StartCPUProfile(f); err != nil {
41+
klog.Errorf("failed to start cpu profile: %v", err)
42+
if err = f.Close(); err != nil {
43+
klog.Errorf("failed to close file %q: %v", path, err)
44+
}
45+
return
46+
}
47+
time.Sleep(30 * time.Second)
48+
pprof.StopCPUProfile()
49+
if err = f.Close(); err != nil {
50+
klog.Errorf("failed to close file %q: %v", path, err)
51+
return
52+
}
53+
}
54+
}()
55+
go func() {
56+
for {
57+
<-ch2
58+
name := fmt.Sprintf("mem-profile-%s.pprof", time.Now().Format(timeFormat))
59+
path := filepath.Join(os.TempDir(), name)
60+
f, err := os.Create(path) // #nosec G303,G304
61+
if err != nil {
62+
klog.Errorf("failed to create memory profile file: %v", err)
63+
return
64+
}
65+
if err = pprof.WriteHeapProfile(f); err != nil {
66+
klog.Errorf("failed to write memory profile file: %v", err)
67+
if err = f.Close(); err != nil {
68+
klog.Errorf("failed to close file %q: %v", path, err)
69+
}
70+
return
71+
}
72+
if err = f.Close(); err != nil {
73+
klog.Errorf("failed to close file %q: %v", path, err)
74+
return
75+
}
76+
}
77+
}()
78+
}
79+
80+
func main() {
81+
cmd := filepath.Base(os.Args[0])
82+
switch cmd {
83+
case CmdController:
84+
dumpProfile()
85+
CmdMain()
86+
case CmdPinger:
87+
dumpProfile()
88+
pinger.CmdMain()
89+
default:
90+
util.LogFatalAndExit(nil, "%s is an unknown command", cmd)
91+
}
92+
}

‎cmd/controller/controller.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package controller
1+
package main
22

33
import (
44
"context"

‎cmd/pinger/pinger.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package main
1+
package pinger
22

33
import (
44
_ "net/http/pprof" // #nosec
@@ -14,7 +14,7 @@ import (
1414
"github.com/kubeovn/kube-ovn/versions"
1515
)
1616

17-
func main() {
17+
func CmdMain() {
1818
defer klog.Flush()
1919

2020
klog.Info(versions.String())

‎dist/images/Dockerfile

+4-4
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,16 @@ COPY 01-kube-ovn.conflist /kube-ovn/01-kube-ovn.conflist
1010
COPY kube-ovn /kube-ovn/kube-ovn
1111
COPY kube-ovn-cmd /kube-ovn/kube-ovn-cmd
1212
COPY kube-ovn-daemon /kube-ovn/kube-ovn-daemon
13-
COPY kube-ovn-pinger /kube-ovn/kube-ovn-pinger
14-
RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \
15-
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-monitor && \
13+
COPY kube-ovn-controller /kube-ovn/kube-ovn-controller
14+
RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-monitor && \
1615
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \
1716
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-webhook && \
1817
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-healthcheck && \
1918
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \
2019
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller && \
20+
ln -s /kube-ovn/kube-ovn-controller /kube-ovn/kube-ovn-pinger && \
2121
setcap CAP_NET_BIND_SERVICE+eip /kube-ovn/kube-ovn-cmd && \
22-
setcap CAP_NET_RAW,CAP_NET_BIND_SERVICE+eip /kube-ovn/kube-ovn-pinger && \
22+
setcap CAP_NET_RAW,CAP_NET_BIND_SERVICE+eip /kube-ovn/kube-ovn-controller && \
2323
setcap CAP_NET_ADMIN,CAP_NET_RAW,CAP_NET_BIND_SERVICE,CAP_SYS_ADMIN+eip /kube-ovn/kube-ovn-daemon
2424

2525
FROM kubeovn/kube-ovn-base:$BASE_TAG

‎dist/images/install.sh

+1
Original file line numberDiff line numberDiff line change
@@ -4733,6 +4733,7 @@ spec:
47334733
capabilities:
47344734
add:
47354735
- NET_BIND_SERVICE
4736+
- NET_RAW
47364737
env:
47374738
- name: ENABLE_SSL
47384739
value: "$ENABLE_SSL"

‎pkg/controller/controller.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1263,7 +1263,7 @@ func (c *Controller) startWorkers(ctx context.Context) {
12631263

12641264
go wait.Until(c.resyncProviderNetworkStatus, 30*time.Second, ctx.Done())
12651265
go wait.Until(c.exportSubnetMetrics, 30*time.Second, ctx.Done())
1266-
go wait.Until(c.CheckGatewayReady, 5*time.Second, ctx.Done())
1266+
go wait.Until(c.checkSubnetGateway, 5*time.Second, ctx.Done())
12671267

12681268
go wait.Until(runWorker("add ovn eip", c.addOvnEipQueue, c.handleAddOvnEip), time.Second, ctx.Done())
12691269
go wait.Until(runWorker("update ovn eip", c.updateOvnEipQueue, c.handleUpdateOvnEip), time.Second, ctx.Done())

‎pkg/controller/node.go

+21-19
Original file line numberDiff line numberDiff line change
@@ -524,27 +524,27 @@ func (c *Controller) handleUpdateNode(key string) error {
524524
return nil
525525
}
526526

527-
func (c *Controller) CheckGatewayReady() {
528-
if err := c.checkGatewayReady(); err != nil {
529-
klog.Errorf("failed to check gateway ready %v", err)
527+
func (c *Controller) checkSubnetGateway() {
528+
if err := c.checkSubnetGatewayNode(); err != nil {
529+
klog.Errorf("failed to check subnet gateway node: %v", err)
530530
}
531531
}
532532

533-
func (c *Controller) checkGatewayReady() error {
534-
klog.V(3).Infoln("start to check gateway status")
533+
func (c *Controller) checkSubnetGatewayNode() error {
534+
klog.V(3).Infoln("start to check subnet gateway node")
535535
subnetList, err := c.subnetsLister.List(labels.Everything())
536536
if err != nil {
537-
klog.Errorf("failed to list subnets %v", err)
537+
klog.Errorf("failed to list subnets: %v", err)
538538
return err
539539
}
540540
nodes, err := c.nodesLister.List(labels.Everything())
541541
if err != nil {
542-
klog.Errorf("failed to list nodes, %v", err)
542+
klog.Errorf("failed to list nodes: %v", err)
543543
return err
544544
}
545545

546546
for _, subnet := range subnetList {
547-
if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) ||
547+
if (subnet.Spec.Vlan != "" && (subnet.Spec.U2OInterconnection || !subnet.Spec.LogicalGateway)) ||
548548
subnet.Spec.GatewayNode == "" ||
549549
subnet.Spec.GatewayType != kubeovnv1.GWCentralizedType ||
550550
!subnet.Spec.EnableEcmp {
@@ -582,24 +582,26 @@ func (c *Controller) checkGatewayReady() error {
582582
pinger.Timeout = time.Duration(count) * time.Second
583583
pinger.Interval = 1 * time.Second
584584

585-
success := false
586-
585+
var pingSucceeded bool
587586
pinger.OnRecv = func(_ *goping.Packet) {
588-
success = true
587+
pingSucceeded = true
589588
pinger.Stop()
590589
}
591590
if err = pinger.Run(); err != nil {
592591
klog.Errorf("failed to run pinger for destination %s: %v", ip, err)
593592
return err
594593
}
595594

596-
if !nodeReady(node) {
597-
success = false
598-
}
599-
600-
if !success {
595+
nodeIsReady := nodeReady(node)
596+
if !pingSucceeded || !nodeIsReady {
601597
if exist {
602-
klog.Warningf("failed to ping ovn0 %s or node %s is not ready, delete ecmp policy route for node", ip, node.Name)
598+
if !pingSucceeded {
599+
klog.Warningf("failed to ping ovn0 ip %s on node %s", ip, node.Name)
600+
}
601+
if !nodeIsReady {
602+
klog.Warningf("node %s is not ready", node.Name)
603+
}
604+
klog.Warningf("delete ecmp policy route for node %s ip %s", node.Name, ip)
603605
nextHops.Remove(ip)
604606
delete(nameIPMap, node.Name)
605607
klog.Infof("update policy route for centralized subnet %s, nextHops %s", subnet.Name, nextHops)
@@ -609,7 +611,7 @@ func (c *Controller) checkGatewayReady() error {
609611
}
610612
}
611613
} else {
612-
klog.V(3).Infof("succeed to ping gw %s", ip)
614+
klog.V(3).Infof("succeeded to ping ovn0 ip %s on node %s", ip, node.Name)
613615
if !exist {
614616
nextHops.Add(ip)
615617
if nameIPMap == nil {
@@ -624,7 +626,7 @@ func (c *Controller) checkGatewayReady() error {
624626
}
625627
}
626628
} else if exist {
627-
klog.Infof("subnet %s gatewayNode does not contains node %v, delete policy route for node ip %s", subnet.Name, node.Name, ip)
629+
klog.Infof("subnet %s gateway nodes does not contain node %s, delete policy route for node ip %s", subnet.Name, node.Name, ip)
628630
nextHops.Remove(ip)
629631
delete(nameIPMap, node.Name)
630632
klog.Infof("update policy route for centralized subnet %s, nextHops %s", subnet.Name, nextHops)

0 commit comments

Comments
 (0)