Skip to content

Commit 2e820fd

Browse files
zbb88888zhangzujian
zbb88888
and
zhangzujian
committed
sbctl chassis operation replace with libovsdb (#3119)
* add ovnsb mod * ovn 层名统一复用用底层 ovs client 命名 * kube-ovn-controller 需要使用 sb nb client, 所以调整命名 * replace sbctl operate chassis Signed-off-by: bobz965 <zhangbingbing2_yewu@cmss.chinamobile.com> Co-authored-by: 张祖建 <zhangzujian.7@gmail.com>
1 parent 00bfa4b commit 2e820fd

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

86 files changed

+4165
-1224
lines changed

‎cmd/daemon/cniserver.go

+15-1
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,21 @@ func initChassisAnno(cfg *daemon.Configuration) error {
164164
}
165165

166166
chassistr := string(chassisID)
167-
node.Annotations[util.ChassisAnnotation] = strings.TrimSpace(chassistr)
167+
chassesName := strings.TrimSpace(chassistr)
168+
if chassesName == "" {
169+
// not ready yet
170+
err = fmt.Errorf("chassis id is empty")
171+
klog.Error(err)
172+
return err
173+
}
174+
if annoChassesName, ok := node.Annotations[util.ChassisAnnotation]; ok {
175+
if annoChassesName == chassesName {
176+
return nil
177+
} else {
178+
klog.Infof("chassis id changed, old: %s, new: %s", annoChassesName, chassesName)
179+
}
180+
}
181+
node.Annotations[util.ChassisAnnotation] = chassesName
168182
patchPayloadTemplate :=
169183
`[{
170184
"op": "%s",

‎mocks/pkg/ovs/interface.go

+753-355
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/controller/controller.go

+14-8
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,9 @@ type Controller struct {
5454
namedPort *NamedPort
5555

5656
ovnLegacyClient *ovs.LegacyClient
57-
ovnClient ovs.OvnClient
57+
58+
ovnNbClient ovs.NbClient
59+
ovnSbClient ovs.SbClient
5860

5961
// ExternalGatewayType define external gateway type, centralized
6062
ExternalGatewayType string
@@ -310,7 +312,7 @@ func Run(ctx context.Context, config *Configuration) {
310312
vpcs: &sync.Map{},
311313
podSubnetMap: &sync.Map{},
312314
deletingPodObjMap: &sync.Map{},
313-
ovnLegacyClient: ovs.NewLegacyClient(config.OvnTimeout, config.OvnSbAddr, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.ClusterTcpSessionLoadBalancer, config.ClusterUdpSessionLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR),
315+
ovnLegacyClient: ovs.NewLegacyClient(config.OvnTimeout),
314316
ipam: ovnipam.NewIPAM(),
315317
namedPort: NewNamedPort(),
316318

@@ -486,10 +488,12 @@ func Run(ctx context.Context, config *Configuration) {
486488
}
487489

488490
var err error
489-
if controller.ovnClient, err = ovs.NewOvnClient(config.OvnNbAddr, config.OvnTimeout, config.NodeSwitchCIDR); err != nil {
490-
util.LogFatalAndExit(err, "failed to create ovn client")
491+
if controller.ovnNbClient, err = ovs.NewOvnNbClient(config.OvnNbAddr, config.OvnTimeout); err != nil {
492+
util.LogFatalAndExit(err, "failed to create ovn nb client")
493+
}
494+
if controller.ovnSbClient, err = ovs.NewOvnSbClient(config.OvnSbAddr, config.OvnTimeout); err != nil {
495+
util.LogFatalAndExit(err, "failed to create ovn sb client")
491496
}
492-
493497
if config.EnableLb {
494498
controller.switchLBRuleLister = switchLBRuleInformer.Lister()
495499
controller.switchLBRuleSynced = switchLBRuleInformer.Informer().HasSynced
@@ -766,11 +770,13 @@ func Run(ctx context.Context, config *Configuration) {
766770
// is closed, at which point it will shutdown the workqueue and wait for
767771
// workers to finish processing their current work items.
768772
func (c *Controller) Run(ctx context.Context) {
769-
if err := c.ovnClient.SetLsDnatModDlDst(c.config.LsDnatModDlDst); err != nil {
773+
// The init process can only be placed here if the init process do really affect the normal process of controller, such as Nodes/Pods/Subnets...
774+
// Otherwise, the init process should be placed after all workers have already started working
775+
if err := c.ovnNbClient.SetLsDnatModDlDst(c.config.LsDnatModDlDst); err != nil {
770776
util.LogFatalAndExit(err, "failed to set NB_Global option ls_dnat_mod_dl_dst")
771777
}
772778

773-
if err := c.ovnClient.SetUseCtInvMatch(); err != nil {
779+
if err := c.ovnNbClient.SetUseCtInvMatch(); err != nil {
774780
util.LogFatalAndExit(err, "failed to set NB_Global option use_ct_inv_match to false")
775781
}
776782

@@ -1148,7 +1154,7 @@ func (c *Controller) startWorkers(ctx context.Context) {
11481154

11491155
func (c *Controller) allSubnetReady(subnets ...string) (bool, error) {
11501156
for _, lsName := range subnets {
1151-
exist, err := c.ovnClient.LogicalSwitchExists(lsName)
1157+
exist, err := c.ovnNbClient.LogicalSwitchExists(lsName)
11521158
if err != nil {
11531159
return false, fmt.Errorf("check logical switch %s exist: %v", lsName, err)
11541160
}

‎pkg/controller/controller_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ type fakeControllerInformers struct {
2626
type fakeController struct {
2727
fakeController *Controller
2828
fakeinformers *fakeControllerInformers
29-
mockOvnClient *mockovs.MockOvnClient
29+
mockOvnClient *mockovs.MockNbClient
3030
}
3131

3232
func alwaysReady() bool { return true }
@@ -50,15 +50,15 @@ func newFakeController(t *testing.T) *fakeController {
5050
}
5151

5252
/* ovn fake client */
53-
mockOvnClient := mockovs.NewMockOvnClient(gomock.NewController(t))
53+
mockOvnClient := mockovs.NewMockNbClient(gomock.NewController(t))
5454

5555
ctrl := &Controller{
5656
servicesLister: serviceInformer.Lister(),
5757
vpcsLister: vpcInformer.Lister(),
5858
vpcSynced: alwaysReady,
5959
subnetsLister: sbunetInformer.Lister(),
6060
subnetSynced: alwaysReady,
61-
ovnClient: mockOvnClient,
61+
ovnNbClient: mockOvnClient,
6262
syncVirtualPortsQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ""),
6363
}
6464

‎pkg/controller/endpoint.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -197,18 +197,18 @@ func (c *Controller) handleUpdateEndpoint(key string) error {
197197
// for performance reason delete lb with no backends
198198
if len(backends) != 0 {
199199
klog.V(3).Infof("update vip %s with backends %s to LB %s", vip, backends, lb)
200-
if err = c.ovnClient.LoadBalancerAddVip(lb, vip, backends...); err != nil {
200+
if err = c.ovnNbClient.LoadBalancerAddVip(lb, vip, backends...); err != nil {
201201
klog.Errorf("failed to add vip %s with backends %s to LB %s: %v", vip, backends, lb, err)
202202
return err
203203
}
204204
} else {
205205
klog.V(3).Infof("delete vip %s from LB %s", vip, lb)
206-
if err := c.ovnClient.LoadBalancerDeleteVip(lb, vip); err != nil {
206+
if err := c.ovnNbClient.LoadBalancerDeleteVip(lb, vip); err != nil {
207207
klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err)
208208
return err
209209
}
210210
klog.V(3).Infof("delete vip %s from old LB %s", vip, lb)
211-
if err := c.ovnClient.LoadBalancerDeleteVip(oldLb, vip); err != nil {
211+
if err := c.ovnNbClient.LoadBalancerDeleteVip(oldLb, vip); err != nil {
212212
klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err)
213213
return err
214214
}

‎pkg/controller/external-gw.go

+14-10
Original file line numberDiff line numberDiff line change
@@ -118,15 +118,15 @@ func (c *Controller) removeExternalGateway() error {
118118

119119
if !keepExternalSubnet {
120120
klog.Infof("delete external gateway switch %s", c.config.ExternalGatewaySwitch)
121-
if err := c.ovnClient.DeleteLogicalGatewaySwitch(util.ExternalGatewaySwitch, c.config.ClusterRouter); err != nil {
121+
if err := c.ovnNbClient.DeleteLogicalGatewaySwitch(util.ExternalGatewaySwitch, c.config.ClusterRouter); err != nil {
122122
klog.Errorf("delete external gateway switch %s: %v", util.ExternalGatewaySwitch, err)
123123
return err
124124
}
125125
} else {
126126
klog.Infof("should keep provider network vlan underlay external gateway switch %s", c.config.ExternalGatewaySwitch)
127127
lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch)
128128
klog.Infof("delete logical router port %s", lrpName)
129-
if err := c.ovnClient.DeleteLogicalRouterPort(lrpName); err != nil {
129+
if err := c.ovnNbClient.DeleteLogicalRouterPort(lrpName); err != nil {
130130
klog.Errorf("failed to delete lrp %s, %v", lrpName, err)
131131
return err
132132
}
@@ -142,7 +142,7 @@ func (c *Controller) establishExternalGateway(config map[string]string) error {
142142
}
143143
var lrpIp, lrpMac string
144144
lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch)
145-
lrp, err := c.ovnClient.GetLogicalRouterPort(lrpName, true)
145+
lrp, err := c.ovnNbClient.GetLogicalRouterPort(lrpName, true)
146146
if err != nil {
147147
klog.Errorf("failed to get lrp %s, %v", lrpName, err)
148148
return err
@@ -160,7 +160,7 @@ func (c *Controller) establishExternalGateway(config map[string]string) error {
160160
lrpIp = config["nic-ip"]
161161
lrpMac = config["nic-mac"]
162162
}
163-
if err := c.ovnClient.CreateGatewayLogicalSwitch(c.config.ExternalGatewaySwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIp, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil {
163+
if err := c.ovnNbClient.CreateGatewayLogicalSwitch(c.config.ExternalGatewaySwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIp, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil {
164164
klog.Errorf("create external gateway switch %s: %v", c.config.ExternalGatewaySwitch, err)
165165
return err
166166
}
@@ -249,15 +249,19 @@ func (c *Controller) getGatewayChassis(config map[string]string) ([]string, erro
249249
klog.Errorf("patch external gw node %s failed %v", gw, err)
250250
return chassises, err
251251
}
252-
chassisID, err := c.ovnLegacyClient.GetChassis(gw)
253-
if err != nil {
254-
klog.Errorf("failed to get external gw %s chassisID, %v", gw, err)
252+
annoChassisName := node.Annotations[util.ChassisAnnotation]
253+
if annoChassisName == "" {
254+
err := fmt.Errorf("node %s has no chassis annotation, kube-ovn-cni not ready", gw)
255+
klog.Error(err)
255256
return chassises, err
256257
}
257-
if chassisID == "" {
258-
return chassises, fmt.Errorf("no chassisID for external gw %s", gw)
258+
klog.Infof("get node %s chassis: %s", gw, annoChassisName)
259+
chassis, err := c.ovnSbClient.GetChassis(annoChassisName, false)
260+
if err != nil {
261+
klog.Errorf("failed to get node %s chassis: %s, %v", node.Name, annoChassisName, err)
262+
return chassises, err
259263
}
260-
chassises = append(chassises, chassisID)
264+
chassises = append(chassises, chassis.UUID)
261265
}
262266
if len(chassises) == 0 {
263267
klog.Error("no available external gw")

‎pkg/controller/external_vpc.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func (c *Controller) syncExternalVpc() {
8383

8484
func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRouter, err error) {
8585
logicalRouters = make(map[string]util.LogicalRouter)
86-
externalOvnRouters, err := c.ovnClient.ListLogicalRouter(false, func(lr *ovnnb.LogicalRouter) bool {
86+
externalOvnRouters, err := c.ovnNbClient.ListLogicalRouter(false, func(lr *ovnnb.LogicalRouter) bool {
8787
return len(lr.ExternalIDs) == 0 || lr.ExternalIDs["vendor"] != util.CniTypeName
8888
})
8989
if err != nil {
@@ -101,7 +101,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
101101
Ports: make([]util.Port, 0, len(externalLR.Ports)),
102102
}
103103
for _, uuid := range externalLR.Ports {
104-
lrp, err := c.ovnClient.GetLogicalRouterPortByUUID(uuid)
104+
lrp, err := c.ovnNbClient.GetLogicalRouterPortByUUID(uuid)
105105
if err != nil {
106106
klog.Warningf("failed to get LRP by UUID %s: %v", uuid, err)
107107
continue
@@ -113,7 +113,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
113113
for routerName, logicalRouter := range logicalRouters {
114114
tmpRouter := logicalRouter
115115
for _, port := range logicalRouter.Ports {
116-
peerPorts, err := c.ovnClient.ListLogicalSwitchPorts(false, nil, func(lsp *ovnnb.LogicalSwitchPort) bool {
116+
peerPorts, err := c.ovnNbClient.ListLogicalSwitchPorts(false, nil, func(lsp *ovnnb.LogicalSwitchPort) bool {
117117
return len(lsp.Options) != 0 && lsp.Options["router-port"] == port.Name
118118
})
119119
if err != nil || len(peerPorts) > 1 {
@@ -124,7 +124,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
124124
continue
125125
}
126126
lsp := peerPorts[0]
127-
switches, err := c.ovnClient.ListLogicalSwitch(false, func(ls *ovnnb.LogicalSwitch) bool {
127+
switches, err := c.ovnNbClient.ListLogicalSwitch(false, func(ls *ovnnb.LogicalSwitch) bool {
128128
return util.ContainsString(ls.Ports, lsp.UUID)
129129
})
130130
if err != nil || len(switches) > 1 {

0 commit comments

Comments
 (0)