underlay: fix NetworkManager operation (#3147)

* underlay: fix NetworkManager operation
* underlay: clear address lifetime

---------

Signed-off-by: 张祖建 <zhangzujian.7@gmail.com>

Author: zhangzujian

go.mod

@@ -35,6 +35,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
 	github.com/vishvananda/netlink v1.2.1-beta.2
+	golang.org/x/mod v0.12.0
 	golang.org/x/sys v0.11.0
 	golang.org/x/time v0.3.0
 	google.golang.org/grpc v1.57.0
@@ -229,7 +230,6 @@ require (
 	go.uber.org/multierr v1.9.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/crypto v0.11.0 // indirect
-	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/net v0.12.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect

go.sum

@@ -1330,8 +1330,8 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
-golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

pkg/daemon/nm_linux.go

@@ -7,6 +7,7 @@ import (
 	"github.com/kubeovn/gonetworkmanager/v2"
 	"github.com/scylladb/go-set/strset"
 	"github.com/vishvananda/netlink"
+	"golang.org/x/mod/semver"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
 )
@@ -190,59 +191,79 @@ func (n *networkManagerSyncer) SetManaged(name string, managed bool) error {
 	}
 
 	if !managed {
-		devices, err := n.manager.GetAllDevices()
+		links, err := netlink.LinkList()
 		if err != nil {
-			klog.Errorf("failed to get all devices from NetworkManager: %v", err)
+			klog.Errorf("failed to list network links: %v", err)
 			return err
 		}
 
-		var hasVlan bool
-		for _, dev := range devices {
-			managed, err := device.GetPropertyManaged()
-			if err != nil {
-				klog.Errorf("failed to get property managed of device %s: %v", dev.GetPath(), err)
-				continue
-			}
-			if !managed {
-				continue
-			}
-
-			devType, err := dev.GetPropertyDeviceType()
-			if err != nil {
-				klog.Errorf("failed to get type of device %s: %v", dev.GetPath(), err)
-				continue
-			}
-			if devType != gonetworkmanager.NmDeviceTypeVlan {
+		for _, l := range links {
+			if l.Attrs().ParentIndex != link.Attrs().Index || l.Type() != "vlan" {
 				continue
 			}
 
-			vlanName, err := dev.GetPropertyIpInterface()
+			d, err := n.manager.GetDeviceByIpIface(l.Attrs().Name)
 			if err != nil {
-				klog.Errorf("failed to get IP interface of device %s: %v", dev.GetPath(), err)
-				continue
+				klog.Errorf("failed to get device by IP iface %q: %v", l.Attrs().Name, err)
+				return err
 			}
-
-			vlanLink, err := netlink.LinkByName(vlanName)
+			vlanManaged, err := d.GetPropertyManaged()
 			if err != nil {
-				klog.Errorf("failed to get link %s: %v", vlanName, err)
+				klog.Errorf("failed to get property managed of device %s: %v", l.Attrs().Name, err)
 				continue
 			}
-			if vlanLink.Type() != "vlan" {
-				klog.Errorf("unexpected link type: %s", vlanLink.Type())
-				continue
+			if vlanManaged {
+				// After setting device managed=no, the vlan interface will be set down by NetworkManager.
+				klog.Infof(`device %q has a vlan interface %q mannaged by NetworkManager, will not set the NetworkManager property "managed" to %v`, name, l.Attrs().Name, managed)
+				return nil
 			}
+		}
 
-			if vlanLink.Attrs().ParentIndex == link.Attrs().Index {
-				klog.Infof("device %s has a vlan interface %s managed by NetworkManager", name, vlanName)
-				hasVlan = true
-				break
-			}
+		version, err := n.manager.GetPropertyVersion()
+		if err != nil {
+			klog.Errorf("failed to get NetworkManager version: %v", err)
+			return err
+		}
+
+		if !strings.HasPrefix(version, "v") {
+			version = "v" + version
 		}
 
-		if hasVlan {
-			klog.Infof(`will not set device %s NetworkManager property "managed" to %v`, name, managed)
+		// Retrieving DNS configuration requires NetworkManager >= v1.6.0.
+		// Do not set device managed=no if the version is < v1.6.0.
+		if semver.Compare(version, "v1.6.0") < 0 {
+			klog.Infof("NetworkManager version %s is less than v1.6.0")
 			return nil
 		}
+
+		// requires NetworkManager >= v1.6
+		dnsManager, err := gonetworkmanager.NewDnsManager()
+		if err != nil {
+			klog.Errorf("failed to initialize NetworkManager DNS manager: %v", err)
+			return err
+		}
+
+		configurations, err := dnsManager.GetPropertyConfiguration()
+		if err != nil {
+			klog.Errorf("failed to get NetworkManager DNS configuration: %v", err)
+			return err
+		}
+
+		for _, c := range configurations {
+			if c.Interface == name {
+				if len(c.Nameservers) != 0 {
+					// After setting device managed=no on CentOS 7 with NetworkManager v1.18.x,
+					// the DNS servers in /etc/resolv.conf configured on the device will be removed.
+					// We don't want to change the host DNS configuration, so skip this operation.
+					klog.Infof("DNS servers %s are configured on interface %s", strings.Join(c.Nameservers, ","), name)
+					if semver.MajorMinor(version) == "v1.18" {
+						klog.Infof("NetworkManager's version is v1.18.x")
+						return nil
+					}
+				}
+				break
+			}
+		}
 	}
 
 	klog.Infof(`setting device %s NetworkManager property "managed" to %v`, name, managed)

pkg/daemon/ovs_linux.go

@@ -955,6 +955,7 @@ func (c *Controller) transferAddrsAndRoutes(nicName, brName string, delNonExiste
 		klog.Infof("address %q has been removed from link %s", addr.String(), nicName)
 
 		addr.Label = ""
+		addr.PreferedLft, addr.ValidLft = 0, 0
 		if err = netlink.AddrReplace(bridge, &addr); err != nil {
 			return 0, fmt.Errorf("failed to replace address %q on OVS bridge %s: %v", addr.String(), brName, err)
 		}