Skip to content

Commit 3dd7f4a

Browse files
zhangzujianzhangzujian
authored
replace ovn legacy client with libovsdb (#3018)
1 parent c5bfdb4 commit 3dd7f4a

22 files changed

+187
-740
lines changed

‎cmd/controller/controller.go

-24
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import (
2222

2323
kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
2424
"github.com/kubeovn/kube-ovn/pkg/controller"
25-
"github.com/kubeovn/kube-ovn/pkg/ovs"
2625
"github.com/kubeovn/kube-ovn/pkg/util"
2726
"github.com/kubeovn/kube-ovn/versions"
2827
)
@@ -54,7 +53,6 @@ func CmdMain() {
5453
util.LogFatalAndExit(err, "failed to check permission")
5554
}
5655

57-
go loopOvnNbctlDaemon(config)
5856
go func() {
5957
mux := http.NewServeMux()
6058
if config.EnableMetrics {
@@ -135,28 +133,6 @@ func CmdMain() {
135133
})
136134
}
137135

138-
func loopOvnNbctlDaemon(config *controller.Configuration) {
139-
for {
140-
daemonSocket := os.Getenv("OVN_NB_DAEMON")
141-
time.Sleep(5 * time.Second)
142-
143-
if _, err := os.Stat(daemonSocket); os.IsNotExist(err) || daemonSocket == "" {
144-
if err := ovs.StartOvnNbctlDaemon(config.OvnNbAddr); err != nil {
145-
klog.Errorf("failed to start ovn-nbctl daemon %v", err)
146-
}
147-
}
148-
149-
// ovn-nbctl daemon may hang and cannot process further request.
150-
// In case of that, we need to start a new daemon.
151-
if err := ovs.CheckAlive(); err != nil {
152-
klog.Warningf("ovn-nbctl daemon doesn't return, start a new daemon")
153-
if err := ovs.StartOvnNbctlDaemon(config.OvnNbAddr); err != nil {
154-
klog.Errorf("failed to start ovn-nbctl daemon %v", err)
155-
}
156-
}
157-
}
158-
}
159-
160136
func checkPermission(config *controller.Configuration) error {
161137
resources := []string{"vpcs", "subnets", "ips", "vlans", "vpc-nat-gateways"}
162138
for _, res := range resources {

‎cmd/controller_health_check/controller_health_check.go

-13
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,10 @@ import (
88
"time"
99

1010
kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
11-
"github.com/kubeovn/kube-ovn/pkg/ovs"
1211
"github.com/kubeovn/kube-ovn/pkg/util"
1312
)
1413

1514
func CmdMain() {
16-
content, err := os.ReadFile("/var/run/ovn/ovn-nbctl.pid")
17-
if err != nil {
18-
util.LogFatalAndExit(err, "failed to get ovn-nbctl daemon pid")
19-
}
20-
daemonPid := strings.TrimSuffix(string(content), "\n")
21-
if err := os.Setenv("OVN_NB_DAEMON", fmt.Sprintf("/var/run/ovn/ovn-nbctl.%s.ctl", daemonPid)); err != nil {
22-
util.LogFatalAndExit(err, "failed to set env OVN_NB_DAEMON")
23-
}
24-
if err := ovs.CheckAlive(); err != nil {
25-
os.Exit(1)
26-
}
27-
2815
addr := "127.0.0.1:10660"
2916
if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
3017
podIpsEnv := os.Getenv("POD_IPS")

‎dist/images/Dockerfile.base

-8
Original file line numberDiff line numberDiff line change
@@ -101,12 +101,6 @@ RUN curl -sSf -L --retry 3 -o /usr/local/bin/bfdd-control https://github.com/bob
101101
curl -sSf -L --retry 3 -o /usr/local/bin/bfdd-beacon https://github.com/bobz965/bfd-binary-for-kube-ovn-cni/releases/download/${BFDD_VERSION}/bfdd-beacon && \
102102
chmod +x /usr/local/bin/bfdd-control /usr/local/bin/bfdd-beacon
103103

104-
ENV DUMB_INIT_VERSION="1.2.5"
105-
RUN dumb_init_arch="x86_64"; \
106-
if [ "$ARCH" = "arm64" ]; then dumb_init_arch="aarch64"; fi; \
107-
curl -sSf -L --retry 5 -o /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_${dumb_init_arch} && \
108-
chmod +x /usr/bin/dumb-init
109-
110104
RUN --mount=type=bind,target=/packages,from=ovs-builder,source=/packages \
111105
dpkg -i /packages/openvswitch-*.deb /packages/python3-openvswitch*.deb && \
112106
dpkg -i --ignore-depends=openvswitch-switch,openvswitch-common /packages/ovn-*.deb && \
@@ -119,5 +113,3 @@ RUN --mount=type=bind,target=/packages,from=ovs-builder,source=/packages \
119113
rm -rf /var/lib/apt/lists/* && \
120114
dpkg -i --ignore-depends=openvswitch-switch,openvswitch-common /packages/*.ddeb; \
121115
fi
122-
123-
ENTRYPOINT ["/usr/bin/dumb-init", "--"]

‎dist/images/start-controller.sh

-23
Original file line numberDiff line numberDiff line change
@@ -32,29 +32,6 @@ function gen_conn_str {
3232
nb_addr="$(gen_conn_str 6641)"
3333
sb_addr="$(gen_conn_str 6642)"
3434

35-
for ((i=0; i<3; i++)); do
36-
if [[ "$ENABLE_SSL" == "false" ]]; then
37-
OVN_NB_DAEMON=$(ovn-nbctl --db="$nb_addr" --pidfile --detach --overwrite-pidfile)
38-
else
39-
OVN_NB_DAEMON=$(ovn-nbctl -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert --db="$nb_addr" --pidfile --detach --overwrite-pidfile)
40-
fi
41-
if echo -n "${OVN_NB_DAEMON}" | grep -qE '^/var/run/ovn/ovn-nbctl\.[0-9]+\.ctl$'; then
42-
export OVN_NB_DAEMON
43-
break
44-
fi
45-
if [ $(echo ${OVN_NB_DAEMON} | wc -c) -gt 64 ]; then
46-
OVN_NB_DAEMON="$(echo ${OVN_NB_DAEMON} | cut -c1-64)..."
47-
fi
48-
echo "invalid ovn-nbctl daemon socket: \"${OVN_NB_DAEMON}\""
49-
unset OVN_NB_DAEMON
50-
pkill -f ovn-nbctl
51-
done
52-
53-
if [ -z "${OVN_NB_DAEMON}" ]; then
54-
echo "failed to start ovn-nbctl daemon"
55-
exit 1
56-
fi
57-
5835
exec ./kube-ovn-controller --ovn-nb-addr="$nb_addr" \
5936
--ovn-sb-addr="$sb_addr" \
6037
$@

‎dist/images/start-webhook.sh

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,2 @@
11
#!/usr/bin/env bash
2-
set -euo pipefail
3-
export OVN_NB_DAEMON=$(ovn-nbctl --db=tcp:${OVN_NB_SERVICE_HOST}:${OVN_NB_SERVICE_PORT} --pidfile --detach)
4-
exec ./kube-ovn-webhook --ovn-nb-host=${OVN_NB_SERVICE_HOST} --ovn-nb-port=${OVN_NB_SERVICE_PORT} $@
2+
exec ./kube-ovn-webhook

‎mocks/pkg/ovs/interface.go

+66-8
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/controller/controller.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ func Run(ctx context.Context, config *Configuration) {
310310
vpcs: &sync.Map{},
311311
podSubnetMap: &sync.Map{},
312312
deletingPodObjMap: &sync.Map{},
313-
ovnLegacyClient: ovs.NewLegacyClient(config.OvnNbAddr, config.OvnTimeout, config.OvnSbAddr, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.ClusterTcpSessionLoadBalancer, config.ClusterUdpSessionLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR),
313+
ovnLegacyClient: ovs.NewLegacyClient(config.OvnTimeout, config.OvnSbAddr, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.ClusterTcpSessionLoadBalancer, config.ClusterUdpSessionLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR),
314314
ipam: ovnipam.NewIPAM(),
315315
namedPort: NewNamedPort(),
316316

‎pkg/controller/external-gw.go

-1
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,6 @@ func (c *Controller) resyncExternalGateway() {
6565
}
6666
exGwEnabled = "true"
6767
lastExGwCM = cm.Data
68-
c.ovnLegacyClient.ExternalGatewayType = cm.Data["type"]
6968
c.ExternalGatewayType = cm.Data["type"]
7069
if err := c.updateDefaultVpcExternal(true); err != nil {
7170
klog.Error("failed to update default vpc, %v", err)

‎pkg/controller/gc.go

+10-8
Original file line numberDiff line numberDiff line change
@@ -119,11 +119,11 @@ func (c *Controller) gcLogicalSwitch() error {
119119
klog.Errorf("failed to list subnet, %v", err)
120120
return err
121121
}
122-
subnetNames := make([]string, 0, len(subnets))
122+
subnetNames := strset.NewWithSize(len(subnets))
123123
subnetMap := make(map[string]*kubeovnv1.Subnet, len(subnets))
124124
for _, s := range subnets {
125125
subnetMap[s.Name] = s
126-
subnetNames = append(subnetNames, s.Name)
126+
subnetNames.Add(s.Name)
127127
}
128128

129129
lss, err := c.ovnClient.ListLogicalSwitch(c.config.EnableExternalVpc, nil)
@@ -152,21 +152,20 @@ func (c *Controller) gcLogicalSwitch() error {
152152
}
153153

154154
klog.Infof("start to gc dhcp options")
155-
dhcpOptions, err := c.ovnLegacyClient.ListDHCPOptions(c.config.EnableExternalVpc, "", "")
155+
dhcpOptions, err := c.ovnClient.ListDHCPOptions(c.config.EnableExternalVpc, nil)
156156
if err != nil {
157157
klog.Errorf("failed to list dhcp options, %v", err)
158158
return err
159159
}
160160
var uuidToDeleteList = []string{}
161161
for _, item := range dhcpOptions {
162-
ls := item.ExternalIds["ls"]
163-
if !util.IsStringIn(ls, subnetNames) {
162+
if len(item.ExternalIDs) == 0 || !subnetNames.Has(item.ExternalIDs["ls"]) {
164163
uuidToDeleteList = append(uuidToDeleteList, item.UUID)
165164
}
166165
}
167166
klog.Infof("gc dhcp options %v", uuidToDeleteList)
168167
if len(uuidToDeleteList) > 0 {
169-
if err = c.ovnLegacyClient.DeleteDHCPOptionsByUUIDs(uuidToDeleteList); err != nil {
168+
if err = c.ovnClient.DeleteDHCPOptionsByUUIDs(uuidToDeleteList...); err != nil {
170169
klog.Errorf("failed to delete dhcp options by uuids, %v", err)
171170
return err
172171
}
@@ -641,11 +640,14 @@ func (c *Controller) gcStaticRoute() error {
641640
continue
642641
}
643642
if route.IPPrefix != "0.0.0.0/0" && route.IPPrefix != "::/0" && c.ipam.ContainAddress(route.IPPrefix) {
644-
exist, err := c.ovnLegacyClient.NatRuleExists(route.IPPrefix)
645-
if exist || err != nil {
643+
exist, err := c.ovnClient.NatExists(c.config.ClusterRouter, "", "", route.IPPrefix)
644+
if err != nil {
646645
klog.Errorf("failed to get NatRule by LogicalIP %s, %v", route.IPPrefix, err)
647646
continue
648647
}
648+
if exist {
649+
continue
650+
}
649651
klog.Infof("gc static route %s %v %s %s", route.RouteTable, route.Policy, route.IPPrefix, route.Nexthop)
650652
if err = c.ovnClient.DeleteLogicalRouterStaticRoute(c.config.ClusterRouter, &route.RouteTable, route.Policy, route.IPPrefix, route.Nexthop); err != nil {
651653
klog.Errorf("failed to delete stale route %s %v %s %s: %v", route.RouteTable, route.Policy, route.IPPrefix, route.Nexthop, err)

‎pkg/controller/ovn_fip.go

+13-8
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7+
"strconv"
78

89
k8serrors "k8s.io/apimachinery/pkg/api/errors"
910
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -14,6 +15,7 @@ import (
1415
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
1516

1617
kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
18+
"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
1719
"github.com/kubeovn/kube-ovn/pkg/util"
1820
)
1921

@@ -251,8 +253,9 @@ func (c *Controller) handleAddOvnFip(key string) error {
251253
return err
252254
}
253255
// ovn add fip
254-
if err = c.ovnLegacyClient.AddFipRule(vpcName, cachedEip.Status.V4Ip,
255-
internalV4Ip, mac, cachedFip.Spec.IpName); err != nil {
256+
if err = c.ovnClient.AddNat(vpcName, ovnnb.NATTypeDNATAndSNAT, cachedEip.Status.V4Ip,
257+
internalV4Ip, mac, cachedFip.Spec.IpName,
258+
map[string]string{"staleless": strconv.FormatBool(c.ExternalGatewayType == kubeovnv1.GWDistributedType)}); err != nil {
256259
klog.Errorf("failed to create v4 fip, %v", err)
257260
return err
258261
}
@@ -327,8 +330,9 @@ func (c *Controller) handleUpdateOvnFip(key string) error {
327330
return err
328331
}
329332
vpcName := subnet.Spec.Vpc
330-
if err = c.ovnLegacyClient.AddFipRule(vpcName, cachedEip.Status.V4Ip,
331-
internalV4Ip, mac, cachedFip.Spec.IpName); err != nil {
333+
if err = c.ovnClient.AddNat(vpcName, ovnnb.NATTypeDNATAndSNAT, cachedEip.Status.V4Ip,
334+
internalV4Ip, mac, cachedFip.Spec.IpName,
335+
map[string]string{"staleless": strconv.FormatBool(c.ExternalGatewayType == kubeovnv1.GWDistributedType)}); err != nil {
332336
klog.Errorf("failed to create v4 fip, %v", err)
333337
return err
334338
}
@@ -347,13 +351,14 @@ func (c *Controller) handleUpdateOvnFip(key string) error {
347351
// fip change eip
348352
if c.ovnFipChangeEip(fip, cachedEip) {
349353
klog.V(3).Infof("fip change ip, old ip '%s', new ip %s", fip.Status.V4Ip, cachedEip.Status.V4Ip)
350-
if err = c.ovnLegacyClient.DeleteFipRule(vpcName, fip.Status.V4Ip, internalV4Ip); err != nil {
354+
if err = c.ovnClient.DeleteNat(vpcName, ovnnb.NATTypeDNATAndSNAT, fip.Status.V4Ip, internalV4Ip); err != nil {
351355
klog.Errorf("failed to create fip, %v", err)
352356
return err
353357
}
354358
// ovn add fip
355-
if err = c.ovnLegacyClient.AddFipRule(vpcName, cachedEip.Status.V4Ip,
356-
internalV4Ip, mac, cachedFip.Spec.IpName); err != nil {
359+
if err = c.ovnClient.AddNat(vpcName, ovnnb.NATTypeDNATAndSNAT, cachedEip.Status.V4Ip,
360+
internalV4Ip, mac, cachedFip.Spec.IpName,
361+
map[string]string{"staleless": strconv.FormatBool(c.ExternalGatewayType == kubeovnv1.GWDistributedType)}); err != nil {
357362
klog.Errorf("failed to create fip, %v", err)
358363
return err
359364
}
@@ -395,7 +400,7 @@ func (c *Controller) handleDelOvnFip(key string) error {
395400
}
396401
// ovn delete fip
397402
if cachedFip.Status.Vpc != "" && cachedFip.Status.V4Eip != "" && cachedFip.Status.V4Ip != "" {
398-
if err = c.ovnLegacyClient.DeleteFipRule(cachedFip.Status.Vpc, cachedFip.Status.V4Eip, cachedFip.Status.V4Ip); err != nil {
403+
if err = c.ovnClient.DeleteNat(cachedFip.Status.Vpc, ovnnb.NATTypeDNATAndSNAT, cachedFip.Status.V4Eip, cachedFip.Status.V4Ip); err != nil {
399404
klog.Errorf("failed to delete fip, %v", err)
400405
return err
401406
}

0 commit comments

Comments
 (0)