underlay: sync NetworkManager IP config to OVS bridge (#2949)

Author: zhangzujian

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/ovn-org/libovsdb v0.0.0-20221101143603-8f21d188c3a5
 	github.com/parnurzeal/gorequest v0.2.16
 	github.com/prometheus/client_golang v1.14.0
+	github.com/scylladb/go-set v1.0.2
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.1

go.sum

@@ -321,6 +321,8 @@ github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
+github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
 github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
@@ -985,6 +987,8 @@ github.com/satori/go.uuid v0.0.0-20180103174451-36e9d2ebbde5/go.mod h1:dA0hQrYB0
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM=
 github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/scylladb/go-set v1.0.2 h1:SkvlMCKhP0wyyct6j+0IHJkBkSZL+TDzZ4E7f7BCcRE=
+github.com/scylladb/go-set v1.0.2/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=

pkg/daemon/controller.go

@@ -73,6 +73,8 @@ type Controller struct {
 	protocol       string
 	localPodName   string
 	localNamespace string
+
+	nmSyncer *networkManagerSyncer
 }
 
 // NewController init a daemon controller
@@ -138,6 +140,9 @@ func NewController(config *Configuration, podInformerFactory informers.SharedInf
 		controller.ipsets[kubeovnv1.ProtocolIPv6] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV6, IPSetPrefix, nil, nil))
 	}
 
+	controller.nmSyncer = newNetworkManagerSyncer()
+	controller.nmSyncer.Run(controller.transferAddrsAndRoutes)
+
 	providerNetworkInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc:    controller.enqueueAddProviderNetwork,
 		UpdateFunc: controller.enqueueUpdateProviderNetwork,
@@ -283,7 +288,7 @@ func (c *Controller) initProviderNetwork(pn *kubeovnv1.ProviderNetwork, node *v1
 
 	var mtu int
 	var err error
-	if mtu, err = ovsInitProviderNetwork(pn.Name, nic, pn.Spec.ExchangeLinkName, c.config.MacLearningFallback); err != nil {
+	if mtu, err = c.ovsInitProviderNetwork(pn.Name, nic, pn.Spec.ExchangeLinkName, c.config.MacLearningFallback); err != nil {
 		if oldLen := len(node.Labels); oldLen != 0 {
 			delete(node.Labels, fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name))
 			delete(node.Labels, fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name))
@@ -397,15 +402,15 @@ func (c *Controller) cleanProviderNetwork(pn *kubeovnv1.ProviderNetwork, node *v
 		return err
 	}
 
-	if err = ovsCleanProviderNetwork(pn.Name); err != nil {
+	if err = c.ovsCleanProviderNetwork(pn.Name); err != nil {
 		return err
 	}
 
 	return nil
 }
 
 func (c *Controller) handleDeleteProviderNetwork(pn *kubeovnv1.ProviderNetwork) error {
-	if err := ovsCleanProviderNetwork(pn.Name); err != nil {
+	if err := c.ovsCleanProviderNetwork(pn.Name); err != nil {
 		return err
 	}
 

pkg/daemon/init.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kubeovn/gonetworkmanager/v2"
 	"github.com/vishvananda/netlink"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
@@ -52,10 +51,6 @@ func InitOVSBridges() (map[string]string, error) {
 					return nil, fmt.Errorf("failed to check vendor of port %s: %v", port, err)
 				}
 				if ok {
-					klog.Infof("config provider nic %s on bridge %s", port, brName)
-					if _, err = configProviderNic(port, brName); err != nil {
-						return nil, err
-					}
 					mappings[port] = brName
 				}
 			}
@@ -109,48 +104,6 @@ func InitMirror(config *Configuration) error {
 	return configureEmptyMirror(config.MirrorNic, config.MTU)
 }
 
-func nmSetManaged(device string, managed bool) error {
-	nm, err := gonetworkmanager.NewNetworkManager()
-	if err != nil {
-		klog.V(5).Infof("failed to connect to NetworkManager: %v", err)
-		return nil
-	}
-
-	running, err := nm.Running()
-	if err != nil {
-		if err != nil {
-			klog.Warningf("failed to check NetworkManager running state: %v", err)
-			return nil
-		}
-	}
-	if !running {
-		klog.V(5).Info("NetworkManager is not running, ignore")
-		return nil
-	}
-
-	d, err := nm.GetDeviceByIpIface(device)
-	if err != nil {
-		klog.Errorf("failed to get device by IP iface %s: %v", device, err)
-		return err
-	}
-	current, err := d.GetPropertyManaged()
-	if err != nil {
-		klog.Errorf("failed to get device property managed: %v", err)
-		return err
-	}
-	if current == managed {
-		return nil
-	}
-
-	klog.Infof(`setting device %s NetworkManager property "managed" to %v`, device, managed)
-	if err = d.SetPropertyManaged(managed); err != nil {
-		klog.Errorf("failed to set device property managed to %v: %v", managed, err)
-		return err
-	}
-
-	return nil
-}
-
 // wait systemd-networkd to finish interface configuration
 func waitNetworkdConfiguration(linkIndex int) {
 	done := make(chan struct{})
@@ -184,7 +137,7 @@ func waitNetworkdConfiguration(linkIndex int) {
 	}
 }
 
-func changeProvideNicName(current, target string) (bool, error) {
+func (c *Controller) changeProvideNicName(current, target string) (bool, error) {
 	link, err := netlink.LinkByName(current)
 	if err != nil {
 		if _, ok := err.(netlink.LinkNotFoundError); ok {
@@ -211,8 +164,8 @@ func changeProvideNicName(current, target string) (bool, error) {
 	}
 
 	// set link unmanaged by NetworkManager
-	if err = nmSetManaged(current, false); err != nil {
-		klog.Errorf("failed set device %s unmanaged by NetworkManager: %v", current, err)
+	if err = c.nmSyncer.SetManaged(current, false); err != nil {
+		klog.Errorf("failed to set device %s unmanaged by NetworkManager: %v", current, err)
 		return false, err
 	}
 
@@ -276,11 +229,11 @@ func changeProvideNicName(current, target string) (bool, error) {
 	return true, nil
 }
 
-func ovsInitProviderNetwork(provider, nic string, exchangeLinkName, macLearningFallback bool) (int, error) {
+func (c *Controller) ovsInitProviderNetwork(provider, nic string, exchangeLinkName, macLearningFallback bool) (int, error) {
 	// create and configure external bridge
 	brName := util.ExternalBridgeName(provider)
 	if exchangeLinkName {
-		exchanged, err := changeProvideNicName(nic, brName)
+		exchanged, err := c.changeProvideNicName(nic, brName)
 		if err != nil {
 			klog.Errorf("failed to change provider nic name from %s to %s: %v", nic, brName, err)
 			return 0, err
@@ -305,7 +258,7 @@ func ovsInitProviderNetwork(provider, nic string, exchangeLinkName, macLearningF
 
 	// add host nic to the external bridge
 	klog.Infof("config provider nic %s on bridge %s", nic, brName)
-	mtu, err := configProviderNic(nic, brName)
+	mtu, err := c.configProviderNic(nic, brName)
 	if err != nil {
 		errMsg := fmt.Errorf("failed to add nic %s to external bridge %s: %v", nic, brName, err)
 		klog.Error(errMsg)
@@ -315,7 +268,7 @@ func ovsInitProviderNetwork(provider, nic string, exchangeLinkName, macLearningF
 	return mtu, nil
 }
 
-func ovsCleanProviderNetwork(provider string) error {
+func (c *Controller) ovsCleanProviderNetwork(provider string) error {
 	mappings, err := getOvnMappings("ovn-bridge-mappings")
 	if err != nil {
 		return err
@@ -368,7 +321,7 @@ func ovsCleanProviderNetwork(provider string) error {
 	klog.V(3).Infof("ovs bridge %s has been deleted", brName)
 
 	if br := util.ExternalBridgeName(provider); br != brName {
-		if _, err = changeProvideNicName(br, brName); err != nil {
+		if _, err = c.changeProvideNicName(br, brName); err != nil {
 			klog.Errorf("failed to change provider nic name from %s to %s: %v", br, brName, err)
 			return err
 		}

pkg/daemon/nm_linux.go

@@ -0,0 +1,186 @@
+package daemon
+
+import (
+	"sync"
+
+	"github.com/kubeovn/gonetworkmanager/v2"
+	"github.com/scylladb/go-set/strset"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+type networkManagerSyncer struct {
+	manager     gonetworkmanager.NetworkManager
+	workqueue   workqueue.Interface
+	devicePaths *strset.Set
+	pathMap     map[string]string
+	bridgeMap   map[string]string
+	lock        sync.Mutex
+}
+
+func newNetworkManagerSyncer() *networkManagerSyncer {
+	syncer := &networkManagerSyncer{}
+
+	manager, err := gonetworkmanager.NewNetworkManager()
+	if err != nil {
+		klog.V(3).Infof("failed to connect to NetworkManager: %v", err)
+		return syncer
+	}
+
+	running, err := manager.Running()
+	if err != nil {
+		klog.V(3).Infof("failed to check NetworkManager running state: %v", err)
+		return syncer
+	}
+	if !running {
+		klog.V(3).Info("NetworkManager is not running, ignore")
+		return syncer
+	}
+
+	syncer.manager = manager
+	syncer.workqueue = workqueue.NewNamed("NetworkManagerSyncer")
+	syncer.devicePaths = strset.New()
+	syncer.pathMap = make(map[string]string)
+	syncer.bridgeMap = make(map[string]string)
+	return syncer
+}
+
+func (n *networkManagerSyncer) Run(handler func(nic, bridge string, delNonExistent bool) (int, error)) {
+	if n.manager == nil {
+		return
+	}
+
+	go func() {
+		for n.ProcessNextItem(handler) {
+		}
+	}()
+
+	go func() {
+		ch := n.manager.Subscribe()
+		defer n.manager.Unsubscribe()
+
+		stateChange := gonetworkmanager.DeviceInterface + "." + gonetworkmanager.ActiveConnectionSignalStateChanged
+		for {
+			event := <-ch
+
+			n.lock.Lock()
+			if len(event.Body) == 0 || event.Name != stateChange || !n.devicePaths.Has(string(event.Path)) {
+				n.lock.Unlock()
+				continue
+			}
+			n.lock.Unlock()
+
+			state, ok := event.Body[0].(uint32)
+			if !ok {
+				klog.Warningf("failed to convert %#v to uint32", event.Body[0])
+				continue
+			}
+			if gonetworkmanager.NmDeviceState(state) != gonetworkmanager.NmDeviceStateActivated {
+				continue
+			}
+
+			klog.Infof("adding dbus object path %s to workqueue", event.Path)
+			n.workqueue.Add(string(event.Path))
+		}
+	}()
+}
+
+func (n *networkManagerSyncer) ProcessNextItem(handler func(nic, bridge string, delNonExistent bool) (int, error)) bool {
+	item, shutdown := n.workqueue.Get()
+	if shutdown {
+		return false
+	}
+	defer n.workqueue.Done(item)
+
+	klog.Infof("process dbus object path %v", item)
+	path := item.(string)
+	n.lock.Lock()
+	if !n.devicePaths.Has(path) {
+		n.lock.Unlock()
+		return true
+	}
+	var nic string
+	for k, v := range n.pathMap {
+		if v == path {
+			nic = k
+			break
+		}
+	}
+	n.lock.Unlock()
+
+	bridge := n.bridgeMap[nic]
+	if _, err := handler(nic, bridge, true); err != nil {
+		klog.Errorf("failed to handle NetworkManager event for device %s with bridge %s: %v", nic, bridge, err)
+	}
+
+	return true
+}
+
+func (n *networkManagerSyncer) AddDevice(nicName, bridge string) error {
+	if n.manager == nil {
+		return nil
+	}
+
+	n.lock.Lock()
+	defer n.lock.Unlock()
+
+	if _, ok := n.pathMap[nicName]; ok {
+		return nil
+	}
+
+	device, err := n.manager.GetDeviceByIpIface(nicName)
+	if err != nil {
+		klog.Errorf("failed to get device by IP iface %q: %v", nicName, err)
+		return err
+	}
+
+	path := string(device.GetPath())
+	klog.V(3).Infof("adding device %s with dbus object path %s and bridge %s", nicName, path, bridge)
+	n.devicePaths.Add(path)
+	n.pathMap[nicName] = path
+	n.bridgeMap[nicName] = bridge
+
+	return nil
+}
+
+func (n *networkManagerSyncer) RemoveDevice(nicName string) error {
+	if n.manager == nil {
+		return nil
+	}
+
+	n.lock.Lock()
+	n.devicePaths.Remove(n.pathMap[nicName])
+	delete(n.pathMap, nicName)
+	delete(n.bridgeMap, nicName)
+	n.lock.Unlock()
+
+	return nil
+}
+
+func (n *networkManagerSyncer) SetManaged(name string, managed bool) error {
+	if n.manager == nil {
+		return nil
+	}
+
+	device, err := n.manager.GetDeviceByIpIface(name)
+	if err != nil {
+		klog.Errorf("failed to get device by IP iface %q: %v", name, err)
+		return err
+	}
+	current, err := device.GetPropertyManaged()
+	if err != nil {
+		klog.Errorf("failed to get device property managed: %v", err)
+		return err
+	}
+	if current == managed {
+		return nil
+	}
+
+	klog.Infof(`setting device %s NetworkManager property "managed" to %v`, name, managed)
+	if err = device.SetPropertyManaged(managed); err != nil {
+		klog.Errorf("failed to set device property managed to %v: %v", managed, err)
+		return err
+	}
+
+	return nil
+}