optimize vm migrate

Signed-off-by: bobz965 <zhangbingbing2_yewu@cmss.chinamobile.com>

Author: zbb88888

mocks/pkg/ovs/interface.go

@@ -641,6 +641,16 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca
 	podName := c.getNameByPod(pod)
 	// todo: isVmPod, getPodType, getNameByPod has duplicated logic
 
+	var err error
+	var isMigrating bool
+	var vmKey, srcNodeName, targetNodeName string
+	if isVMPod && c.config.EnableKeepVMIP {
+		vmKey = fmt.Sprintf("%s/%s", namespace, vmName)
+		if isMigrating, srcNodeName, targetNodeName, err = c.migrateVM(pod, vmKey); err != nil {
+			klog.Error(err)
+			return nil, err
+		}
+	}
 	// Avoid create lsp for already running pod in ovn-nb when controller restart
 	for _, podNet := range needAllocatePodNets {
 		// the subnet may changed when alloc static ip from the latter subnet after ns supports multi subnets
@@ -669,10 +679,11 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca
 			delete(pod.Annotations, fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName))
 		}
 		pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true"
-		if isVMPod && c.config.EnableKeepVMIP {
+
+		if vmKey != "" {
 			pod.Annotations[fmt.Sprintf(util.VMTemplate, podNet.ProviderName)] = vmName
 			if err := c.changeVMSubnet(vmName, namespace, podNet.ProviderName, subnet.Name); err != nil {
-				klog.Errorf("change subnet of pod %s/%s to %s failed: %v", namespace, name, subnet.Name, err)
+				klog.Errorf("vm %s change subnet to %s failed: %v", vmKey, subnet.Name, err)
 				return nil, err
 			}
 		}
@@ -720,12 +731,31 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca
 				DHCPv6OptionsUUID: subnet.Status.DHCPv6OptionsUUID,
 			}
 
-			if err := c.OVNNbClient.CreateLogicalSwitchPort(subnet.Name, portName, ipStr, mac, podName, pod.Namespace, portSecurity, securityGroupAnnotation, vips, podNet.Subnet.Spec.EnableDHCP, dhcpOptions, subnet.Spec.Vpc); err != nil {
+			if err := c.OVNNbClient.CreateLogicalSwitchPort(subnet.Name, portName, ipStr, mac, podName, pod.Namespace,
+				portSecurity, securityGroupAnnotation, vips, podNet.Subnet.Spec.EnableDHCP, dhcpOptions, subnet.Spec.Vpc); err != nil {
 				c.recorder.Eventf(pod, v1.EventTypeWarning, "CreateOVNPortFailed", err.Error())
 				klog.Errorf("%v", err)
 				return nil, err
 			}
 
+			if vmKey != "" {
+				if isMigrating {
+					if err = c.setVMLSPMigrationOptions(portName, srcNodeName, targetNodeName); err != nil {
+						klog.Error(err)
+						return nil, err
+					}
+					pod.Annotations[util.OVNMigrationAnnotation] = "true"
+				} else {
+					if migrate, ok := pod.Annotations[util.OVNMigrationAnnotation]; ok && migrate == "true" {
+						if err = c.cleanVMLSPMigrationOptions(portName); err != nil {
+							klog.Error(err)
+							return nil, err
+						}
+						pod.Annotations[util.OVNMigratedAnnotation] = "true"
+					}
+				}
+			}
+
 			if pod.Annotations[fmt.Sprintf(util.Layer2ForwardAnnotationTemplate, podNet.ProviderName)] == "true" {
 				if err := c.OVNNbClient.EnablePortLayer2forward(portName); err != nil {
 					c.recorder.Eventf(pod, v1.EventTypeWarning, "SetOVNPortL2ForwardFailed", err.Error())
@@ -2089,3 +2119,72 @@ func (c *Controller) getVirtualIPs(pod *v1.Pod, podNets []*kubeovnNet) map[strin
 	}
 	return vipsMap
 }
+
+// migrate vm return migrate, src node, target node, err
+func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, string, string, error) {
+	if migrated, ok := pod.Annotations[util.OVNMigratedAnnotation]; ok && migrated == "true" {
+		klog.Infof("vm %s is migrated", vmKey)
+		return false, "", "", nil
+	}
+
+	migratePhase, ok := pod.Annotations[util.MigrationPhaseAnnotation]
+	if !ok {
+		// not migrate vm
+		return false, "", "", nil
+	}
+	// check migrate phase
+	if migratePhase == "" {
+		err := fmt.Errorf("vm %s migration phase is empty", vmKey)
+		klog.Error(err)
+		return false, "", "", err
+	}
+	if migratePhase == util.MigrationStateSucceeded {
+		klog.Infof("vm %s migration is succeeded", vmKey)
+		return false, "", "", nil
+	}
+	if migratePhase == util.MigrationStateFailed {
+		klog.Warningf("vm %s migration is failed", vmKey)
+		return false, "", "", nil
+	}
+	if migratePhase == util.MigrationStateStarted {
+		if isTarget, ok := pod.Annotations[util.MigrationTargetAnnotation]; ok && isTarget == "true" {
+			klog.Infof("start to migrate target vm %s", vmKey)
+			// this pod is target vm pod
+			srcNode, ok := pod.Annotations[util.MigrationSourceNodeAnnotation]
+			if !ok || srcNode == "" {
+				err := fmt.Errorf("vm %s migration source node is not set", vmKey)
+				klog.Error(err)
+				return false, "", "", err
+			}
+			targetNode := pod.Spec.NodeName
+			if targetNode == "" {
+				err := fmt.Errorf("vm %s migration target node is not set", vmKey)
+				klog.Error(err)
+				return false, "", "", err
+			}
+			return true, srcNode, targetNode, nil
+		}
+		klog.Infof("start to migrate src vm %s", vmKey)
+	}
+	return false, "", "", nil
+}
+
+func (c *Controller) setVMLSPMigrationOptions(portName, srcNodeName, targetNodeName string) error {
+	klog.Infof("set migrate options for lsp %s", portName)
+	if err := c.OVNNbClient.SetLogicalSwitchPortMigrateOptions(portName, srcNodeName, targetNodeName); err != nil {
+		err = fmt.Errorf("failed to set migrate options for lsp %s, %v", portName, err)
+		klog.Error(err)
+		return err
+	}
+	return nil
+}
+
+func (c *Controller) cleanVMLSPMigrationOptions(portName string) error {
+	klog.Infof("clean migrate options for lsp %s", portName)
+	if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(portName); err != nil {
+		err = fmt.Errorf("failed to clean migrate options for lsp %s, %v", portName, err)
+		klog.Error(err)
+		return err
+	}
+	return nil
+}

pkg/controller/pod.go

@@ -91,6 +91,9 @@ type LogicalSwitchPort interface {
 	ListLogicalSwitchPortsWithLegacyExternalIDs() ([]ovnnb.LogicalSwitchPort, error)
 	GetLogicalSwitchPort(lspName string, ignoreNotFound bool) (*ovnnb.LogicalSwitchPort, error)
 	LogicalSwitchPortExists(name string) (bool, error)
+	// vm live migrate
+	SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string) error
+	CleanLogicalSwitchPortMigrateOptions(lspName string) error
 }
 
 type LoadBalancer interface {

pkg/ovs/interface.go

@@ -795,3 +795,85 @@ func getLogicalSwitchPortSgs(lsp *ovnnb.LogicalSwitchPort) *strset.Set {
 
 	return sgs
 }
+
+// SetLogicalSwitchPortMigrateOptions set logical switch port options of migrate
+func (c *OVNNbClient) SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string) error {
+	// to facilitate the migration of the VM: ovn-nbctl lsp-set-options migrator requested-chassis=src,target activation-strategy=rarp
+	// the options will be removed after the migration is completed
+	if srcNodeName == targetNodeName {
+		err := fmt.Errorf("src and target node can not be the same on migrator port %s", lspName)
+		klog.Error(err)
+		return err
+	}
+	if srcNodeName == "" || targetNodeName == "" {
+		err := fmt.Errorf("src and target node can not be empty on migrator port %s", lspName)
+		klog.Error(err)
+		return err
+	}
+
+	lsp, src, target, err := c.GetLogicalSwitchPortMigrateOptions(lspName)
+	if err != nil {
+		klog.Error(err)
+		return err
+	}
+	if src == srcNodeName && target == targetNodeName {
+		// already set
+		return nil
+	}
+
+	requestedChassis := fmt.Sprintf("%s,%s", srcNodeName, targetNodeName)
+	klog.Infof("set logical switch port %s options requested-chassis=%s", lspName, requestedChassis)
+	if lsp.Options == nil {
+		lsp.Options = make(map[string]string)
+	}
+	lsp.Options["requested-chassis"] = requestedChassis
+	lsp.Options["activation-strategy"] = "rarp"
+	if err := c.UpdateLogicalSwitchPort(lsp, &lsp.Options); err != nil {
+		err = fmt.Errorf("failed to set logical switch port %s options requested chassis %s: %v", lspName, requestedChassis, err)
+		klog.Error(err)
+		return err
+	}
+	return nil
+}
+
+// GetLogicalSwitchPortMigrateOptions get logical switch port src and target node name options of migrate
+func (c *OVNNbClient) GetLogicalSwitchPortMigrateOptions(lspName string) (*ovnnb.LogicalSwitchPort, string, string, error) {
+	lsp, err := c.GetLogicalSwitchPort(lspName, true)
+	if err != nil {
+		err = fmt.Errorf("failed to get logical switch port %s: %v", lspName, err)
+		klog.Error(err)
+		return nil, "", "", err
+	}
+	if lsp == nil || lsp.Options == nil {
+		return nil, "", "", nil
+	}
+
+	requestedChassis, ok := lsp.Options["requested-chassis"]
+	if ok {
+		splits := strings.Split(requestedChassis, ",")
+		if len(splits) == 2 {
+			return lsp, splits[0], splits[1], nil
+		}
+	}
+	return nil, "", "", nil
+}
+
+// CleanLogicalSwitchPortMigrateOptions clean logical switch port options of migration
+func (c *OVNNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string) error {
+	lsp, src, target, err := c.GetLogicalSwitchPortMigrateOptions(lspName)
+	if err != nil {
+		klog.Error(err)
+		return err
+	}
+	if src != "" && target != "" {
+		lsp.Options = make(map[string]string)
+		lsp.Options["requested-chassis"] = target
+		klog.Infof("clean logical switch port %s options", lspName)
+		if err := c.UpdateLogicalSwitchPort(lsp, &lsp.Options); err != nil {
+			err = fmt.Errorf("failed to clean options for logical switch port %s : %v", lspName, err)
+			klog.Error(err)
+			return err
+		}
+	}
+	return nil
+}

pkg/ovs/ovn-nb-logical_switch_port.go

@@ -292,8 +292,13 @@ const (
 	ConsumptionKubevirt       = "kubevirt"
 	VhostUserSocketVolumeName = "vhostuser-sockets"
 
+	OVNMigrationAnnotation        = "ovn.kubernetes.io/migrate"         // to control if ovn set lsp migrate options
+	OVNMigratedAnnotation         = "ovn.kubernetes.io/migrated"        // to control if ovn clean lsp migrate options
 	MigrationSourceNodeAnnotation = "kubevirt.io/migration-source-node" // target pod has source node name
 	MigrationSourceAnnotation     = "kubevirt.io/migration-source"      // migration source vm: true or false
 	MigrationTargetAnnotation     = "kubevirt.io/migration-target"      // migration target vm: true or false
 	MigrationPhaseAnnotation      = "kubevirt.io/migration-phase"       // migration vm phase: started/succeeded/failed
+	MigrationStateStarted         = "started"
+	MigrationStateSucceeded       = "succeeded"
+	MigrationStateFailed          = "failed"
 )