From 3348589ff26e78d5e43f7dd867044a45e236de78 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Fri, 23 Feb 2024 17:38:24 +0800 Subject: [PATCH 01/12] add kubevirt vm migration annotation Signed-off-by: bobz965 --- pkg/util/const.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/util/const.go b/pkg/util/const.go index 6a391d89f4f..9553354d42f 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -291,4 +291,9 @@ const ( ConsumptionKubevirt = "kubevirt" VhostUserSocketVolumeName = "vhostuser-sockets" + + MigrationSourceNodeAnnotation = "kubevirt.io/migration-source-node" // target pod has source node name + MigrationSourceAnnotation = "kubevirt.io/migration-source" // migration source vm: true or false + MigrationTargetAnnotation = "kubevirt.io/migration-target" // migration target vm: true or false + MigrationPhaseAnnotation = "kubevirt.io/migration-phase" // migration vm phase: started/succeeded/failed ) From f8258851e3a3da7036e3b30732e053af635be8d0 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Mon, 26 Feb 2024 17:38:16 +0800 Subject: [PATCH 02/12] optimize vm migrate Signed-off-by: bobz965 --- mocks/pkg/ovs/interface.go | 56 ++++++++++++++ pkg/controller/pod.go | 105 +++++++++++++++++++++++++- pkg/ovs/interface.go | 3 + pkg/ovs/ovn-nb-logical_switch_port.go | 82 ++++++++++++++++++++ pkg/util/const.go | 5 ++ 5 files changed, 248 insertions(+), 3 deletions(-) diff --git a/mocks/pkg/ovs/interface.go b/mocks/pkg/ovs/interface.go index 7031e89985f..199f0ccd993 100644 --- a/mocks/pkg/ovs/interface.go +++ b/mocks/pkg/ovs/interface.go @@ -767,6 +767,20 @@ func (m *MockLogicalSwitchPort) EXPECT() *MockLogicalSwitchPortMockRecorder { return m.recorder } +// CleanLogicalSwitchPortMigrateOptions mocks base method. +func (m *MockLogicalSwitchPort) CleanLogicalSwitchPortMigrateOptions(lspName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CleanLogicalSwitchPortMigrateOptions", lspName) + ret0, _ := ret[0].(error) + return ret0 +} + +// CleanLogicalSwitchPortMigrateOptions indicates an expected call of CleanLogicalSwitchPortMigrateOptions. +func (mr *MockLogicalSwitchPortMockRecorder) CleanLogicalSwitchPortMigrateOptions(lspName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockLogicalSwitchPort)(nil).CleanLogicalSwitchPortMigrateOptions), lspName) +} + // CreateBareLogicalSwitchPort mocks base method. func (m *MockLogicalSwitchPort) CreateBareLogicalSwitchPort(lsName, lspName, ip, mac string) error { m.ctrl.T.Helper() @@ -987,6 +1001,20 @@ func (mr *MockLogicalSwitchPortMockRecorder) SetLogicalSwitchPortExternalIDs(lsp return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLogicalSwitchPortExternalIDs", reflect.TypeOf((*MockLogicalSwitchPort)(nil).SetLogicalSwitchPortExternalIDs), lspName, externalIDs) } +// SetLogicalSwitchPortMigrateOptions mocks base method. +func (m *MockLogicalSwitchPort) SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetLogicalSwitchPortMigrateOptions", lspName, srcNodeName, targetNodeName) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetLogicalSwitchPortMigrateOptions indicates an expected call of SetLogicalSwitchPortMigrateOptions. +func (mr *MockLogicalSwitchPortMockRecorder) SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockLogicalSwitchPort)(nil).SetLogicalSwitchPortMigrateOptions), lspName, srcNodeName, targetNodeName) +} + // SetLogicalSwitchPortSecurity mocks base method. func (m *MockLogicalSwitchPort) SetLogicalSwitchPortSecurity(portSecurity bool, lspName, mac, ips, vips string) error { m.ctrl.T.Helper() @@ -2434,6 +2462,20 @@ func (mr *MockNbClientMockRecorder) AddressSetUpdateAddress(asName any, addresse return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddressSetUpdateAddress", reflect.TypeOf((*MockNbClient)(nil).AddressSetUpdateAddress), varargs...) } +// CleanLogicalSwitchPortMigrateOptions mocks base method. +func (m *MockNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CleanLogicalSwitchPortMigrateOptions", lspName) + ret0, _ := ret[0].(error) + return ret0 +} + +// CleanLogicalSwitchPortMigrateOptions indicates an expected call of CleanLogicalSwitchPortMigrateOptions. +func (mr *MockNbClientMockRecorder) CleanLogicalSwitchPortMigrateOptions(lspName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockNbClient)(nil).CleanLogicalSwitchPortMigrateOptions), lspName) +} + // ClearLogicalRouterPolicy mocks base method. func (m *MockNbClient) ClearLogicalRouterPolicy(lrName string) error { m.ctrl.T.Helper() @@ -4072,6 +4114,20 @@ func (mr *MockNbClientMockRecorder) SetLogicalSwitchPortExternalIDs(lspName, ext return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLogicalSwitchPortExternalIDs", reflect.TypeOf((*MockNbClient)(nil).SetLogicalSwitchPortExternalIDs), lspName, externalIDs) } +// SetLogicalSwitchPortMigrateOptions mocks base method. +func (m *MockNbClient) SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetLogicalSwitchPortMigrateOptions", lspName, srcNodeName, targetNodeName) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetLogicalSwitchPortMigrateOptions indicates an expected call of SetLogicalSwitchPortMigrateOptions. +func (mr *MockNbClientMockRecorder) SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockNbClient)(nil).SetLogicalSwitchPortMigrateOptions), lspName, srcNodeName, targetNodeName) +} + // SetLogicalSwitchPortSecurity mocks base method. func (m *MockNbClient) SetLogicalSwitchPortSecurity(portSecurity bool, lspName, mac, ips, vips string) error { m.ctrl.T.Helper() diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 1a91d7fd474..b8b745f0304 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -641,6 +641,16 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca podName := c.getNameByPod(pod) // todo: isVmPod, getPodType, getNameByPod has duplicated logic + var err error + var isMigrating bool + var vmKey, srcNodeName, targetNodeName string + if isVMPod && c.config.EnableKeepVMIP { + vmKey = fmt.Sprintf("%s/%s", namespace, vmName) + if isMigrating, srcNodeName, targetNodeName, err = c.migrateVM(pod, vmKey); err != nil { + klog.Error(err) + return nil, err + } + } // Avoid create lsp for already running pod in ovn-nb when controller restart for _, podNet := range needAllocatePodNets { // the subnet may changed when alloc static ip from the latter subnet after ns supports multi subnets @@ -669,10 +679,11 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca delete(pod.Annotations, fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName)) } pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true" - if isVMPod && c.config.EnableKeepVMIP { + + if vmKey != "" { pod.Annotations[fmt.Sprintf(util.VMTemplate, podNet.ProviderName)] = vmName if err := c.changeVMSubnet(vmName, namespace, podNet.ProviderName, subnet.Name); err != nil { - klog.Errorf("change subnet of pod %s/%s to %s failed: %v", namespace, name, subnet.Name, err) + klog.Errorf("vm %s change subnet to %s failed: %v", vmKey, subnet.Name, err) return nil, err } } @@ -720,12 +731,31 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca DHCPv6OptionsUUID: subnet.Status.DHCPv6OptionsUUID, } - if err := c.OVNNbClient.CreateLogicalSwitchPort(subnet.Name, portName, ipStr, mac, podName, pod.Namespace, portSecurity, securityGroupAnnotation, vips, podNet.Subnet.Spec.EnableDHCP, dhcpOptions, subnet.Spec.Vpc); err != nil { + if err := c.OVNNbClient.CreateLogicalSwitchPort(subnet.Name, portName, ipStr, mac, podName, pod.Namespace, + portSecurity, securityGroupAnnotation, vips, podNet.Subnet.Spec.EnableDHCP, dhcpOptions, subnet.Spec.Vpc); err != nil { c.recorder.Eventf(pod, v1.EventTypeWarning, "CreateOVNPortFailed", err.Error()) klog.Errorf("%v", err) return nil, err } + if vmKey != "" { + if isMigrating { + if err = c.setVMLSPMigrationOptions(portName, srcNodeName, targetNodeName); err != nil { + klog.Error(err) + return nil, err + } + pod.Annotations[util.OVNMigrationAnnotation] = "true" + } else { + if migrate, ok := pod.Annotations[util.OVNMigrationAnnotation]; ok && migrate == "true" { + if err = c.cleanVMLSPMigrationOptions(portName); err != nil { + klog.Error(err) + return nil, err + } + pod.Annotations[util.OVNMigratedAnnotation] = "true" + } + } + } + if pod.Annotations[fmt.Sprintf(util.Layer2ForwardAnnotationTemplate, podNet.ProviderName)] == "true" { if err := c.OVNNbClient.EnablePortLayer2forward(portName); err != nil { c.recorder.Eventf(pod, v1.EventTypeWarning, "SetOVNPortL2ForwardFailed", err.Error()) @@ -2089,3 +2119,72 @@ func (c *Controller) getVirtualIPs(pod *v1.Pod, podNets []*kubeovnNet) map[strin } return vipsMap } + +// migrate vm return migrate, src node, target node, err +func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, string, string, error) { + if migrated, ok := pod.Annotations[util.OVNMigratedAnnotation]; ok && migrated == "true" { + klog.Infof("vm %s is migrated", vmKey) + return false, "", "", nil + } + + migratePhase, ok := pod.Annotations[util.MigrationPhaseAnnotation] + if !ok { + // not migrate vm + return false, "", "", nil + } + // check migrate phase + if migratePhase == "" { + err := fmt.Errorf("vm %s migration phase is empty", vmKey) + klog.Error(err) + return false, "", "", err + } + if migratePhase == util.MigrationStateSucceeded { + klog.Infof("vm %s migration is succeeded", vmKey) + return false, "", "", nil + } + if migratePhase == util.MigrationStateFailed { + klog.Warningf("vm %s migration is failed", vmKey) + return false, "", "", nil + } + if migratePhase == util.MigrationStateStarted { + if isTarget, ok := pod.Annotations[util.MigrationTargetAnnotation]; ok && isTarget == "true" { + klog.Infof("start to migrate target vm %s", vmKey) + // this pod is target vm pod + srcNode, ok := pod.Annotations[util.MigrationSourceNodeAnnotation] + if !ok || srcNode == "" { + err := fmt.Errorf("vm %s migration source node is not set", vmKey) + klog.Error(err) + return false, "", "", err + } + targetNode := pod.Spec.NodeName + if targetNode == "" { + err := fmt.Errorf("vm %s migration target node is not set", vmKey) + klog.Error(err) + return false, "", "", err + } + return true, srcNode, targetNode, nil + } + klog.Infof("start to migrate src vm %s", vmKey) + } + return false, "", "", nil +} + +func (c *Controller) setVMLSPMigrationOptions(portName, srcNodeName, targetNodeName string) error { + klog.Infof("set migrate options for lsp %s", portName) + if err := c.OVNNbClient.SetLogicalSwitchPortMigrateOptions(portName, srcNodeName, targetNodeName); err != nil { + err = fmt.Errorf("failed to set migrate options for lsp %s, %v", portName, err) + klog.Error(err) + return err + } + return nil +} + +func (c *Controller) cleanVMLSPMigrationOptions(portName string) error { + klog.Infof("clean migrate options for lsp %s", portName) + if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(portName); err != nil { + err = fmt.Errorf("failed to clean migrate options for lsp %s, %v", portName, err) + klog.Error(err) + return err + } + return nil +} diff --git a/pkg/ovs/interface.go b/pkg/ovs/interface.go index 4c958e3874b..10c1e41ef65 100644 --- a/pkg/ovs/interface.go +++ b/pkg/ovs/interface.go @@ -91,6 +91,9 @@ type LogicalSwitchPort interface { ListLogicalSwitchPortsWithLegacyExternalIDs() ([]ovnnb.LogicalSwitchPort, error) GetLogicalSwitchPort(lspName string, ignoreNotFound bool) (*ovnnb.LogicalSwitchPort, error) LogicalSwitchPortExists(name string) (bool, error) + // vm live migrate + SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string) error + CleanLogicalSwitchPortMigrateOptions(lspName string) error } type LoadBalancer interface { diff --git a/pkg/ovs/ovn-nb-logical_switch_port.go b/pkg/ovs/ovn-nb-logical_switch_port.go index 336ef34dabf..0aeed7a76e8 100644 --- a/pkg/ovs/ovn-nb-logical_switch_port.go +++ b/pkg/ovs/ovn-nb-logical_switch_port.go @@ -799,3 +799,85 @@ func getLogicalSwitchPortSgs(lsp *ovnnb.LogicalSwitchPort) *strset.Set { return sgs } + +// SetLogicalSwitchPortMigrateOptions set logical switch port options of migrate +func (c *OVNNbClient) SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string) error { + // to facilitate the migration of the VM: ovn-nbctl lsp-set-options migrator requested-chassis=src,target activation-strategy=rarp + // the options will be removed after the migration is completed + if srcNodeName == "" || targetNodeName == "" { + err := fmt.Errorf("src and target node can not be empty on migrator port %s", lspName) + klog.Error(err) + return err + } + if srcNodeName == targetNodeName { + err := fmt.Errorf("src and target node can not be the same on migrator port %s", lspName) + klog.Error(err) + return err + } + + lsp, src, target, err := c.GetLogicalSwitchPortMigrateOptions(lspName) + if err != nil { + klog.Error(err) + return err + } + if src == srcNodeName && target == targetNodeName { + // already set + return nil + } + + requestedChassis := fmt.Sprintf("%s,%s", srcNodeName, targetNodeName) + klog.Infof("set logical switch port %s options requested-chassis=%s", lspName, requestedChassis) + if lsp.Options == nil { + lsp.Options = make(map[string]string) + } + lsp.Options["requested-chassis"] = requestedChassis + lsp.Options["activation-strategy"] = "rarp" + if err := c.UpdateLogicalSwitchPort(lsp, &lsp.Options); err != nil { + err = fmt.Errorf("failed to set logical switch port %s options requested chassis %s: %v", lspName, requestedChassis, err) + klog.Error(err) + return err + } + return nil +} + +// GetLogicalSwitchPortMigrateOptions get logical switch port src and target node name options of migrate +func (c *OVNNbClient) GetLogicalSwitchPortMigrateOptions(lspName string) (*ovnnb.LogicalSwitchPort, string, string, error) { + lsp, err := c.GetLogicalSwitchPort(lspName, true) + if err != nil { + err = fmt.Errorf("failed to get logical switch port %s: %v", lspName, err) + klog.Error(err) + return nil, "", "", err + } + if lsp == nil || lsp.Options == nil { + return nil, "", "", nil + } + + requestedChassis, ok := lsp.Options["requested-chassis"] + if ok { + splits := strings.Split(requestedChassis, ",") + if len(splits) == 2 { + return lsp, splits[0], splits[1], nil + } + } + return nil, "", "", nil +} + +// CleanLogicalSwitchPortMigrateOptions clean logical switch port options of migration +func (c *OVNNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string) error { + lsp, src, target, err := c.GetLogicalSwitchPortMigrateOptions(lspName) + if err != nil { + klog.Error(err) + return err + } + if src != "" && target != "" { + lsp.Options = make(map[string]string) + lsp.Options["requested-chassis"] = target + klog.Infof("clean logical switch port %s options", lspName) + if err := c.UpdateLogicalSwitchPort(lsp, &lsp.Options); err != nil { + err = fmt.Errorf("failed to clean options for logical switch port %s : %v", lspName, err) + klog.Error(err) + return err + } + } + return nil +} diff --git a/pkg/util/const.go b/pkg/util/const.go index 9553354d42f..1213f934194 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -292,8 +292,13 @@ const ( ConsumptionKubevirt = "kubevirt" VhostUserSocketVolumeName = "vhostuser-sockets" + OVNMigrationAnnotation = "ovn.kubernetes.io/migrate" // to control if ovn set lsp migrate options + OVNMigratedAnnotation = "ovn.kubernetes.io/migrated" // to control if ovn clean lsp migrate options MigrationSourceNodeAnnotation = "kubevirt.io/migration-source-node" // target pod has source node name MigrationSourceAnnotation = "kubevirt.io/migration-source" // migration source vm: true or false MigrationTargetAnnotation = "kubevirt.io/migration-target" // migration target vm: true or false MigrationPhaseAnnotation = "kubevirt.io/migration-phase" // migration vm phase: started/succeeded/failed + MigrationStateStarted = "started" + MigrationStateSucceeded = "succeeded" + MigrationStateFailed = "failed" ) From a2c07b0939ed5a78fc97faf43052cd9f1b641642 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Thu, 29 Feb 2024 11:23:03 +0800 Subject: [PATCH 03/12] consider rollback Signed-off-by: bobz965 --- mocks/pkg/ovs/interface.go | 16 ++--- pkg/controller/pod.go | 97 +++++++++++++-------------- pkg/ovs/interface.go | 2 +- pkg/ovs/ovn-nb-logical_switch_port.go | 13 +++- pkg/util/const.go | 1 - 5 files changed, 67 insertions(+), 62 deletions(-) diff --git a/mocks/pkg/ovs/interface.go b/mocks/pkg/ovs/interface.go index 199f0ccd993..1aee8f0640a 100644 --- a/mocks/pkg/ovs/interface.go +++ b/mocks/pkg/ovs/interface.go @@ -768,17 +768,17 @@ func (m *MockLogicalSwitchPort) EXPECT() *MockLogicalSwitchPortMockRecorder { } // CleanLogicalSwitchPortMigrateOptions mocks base method. -func (m *MockLogicalSwitchPort) CleanLogicalSwitchPortMigrateOptions(lspName string) error { +func (m *MockLogicalSwitchPort) CleanLogicalSwitchPortMigrateOptions(lspName string, fail bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CleanLogicalSwitchPortMigrateOptions", lspName) + ret := m.ctrl.Call(m, "CleanLogicalSwitchPortMigrateOptions", lspName, fail) ret0, _ := ret[0].(error) return ret0 } // CleanLogicalSwitchPortMigrateOptions indicates an expected call of CleanLogicalSwitchPortMigrateOptions. -func (mr *MockLogicalSwitchPortMockRecorder) CleanLogicalSwitchPortMigrateOptions(lspName any) *gomock.Call { +func (mr *MockLogicalSwitchPortMockRecorder) CleanLogicalSwitchPortMigrateOptions(lspName, fail any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockLogicalSwitchPort)(nil).CleanLogicalSwitchPortMigrateOptions), lspName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockLogicalSwitchPort)(nil).CleanLogicalSwitchPortMigrateOptions), lspName, fail) } // CreateBareLogicalSwitchPort mocks base method. @@ -2463,17 +2463,17 @@ func (mr *MockNbClientMockRecorder) AddressSetUpdateAddress(asName any, addresse } // CleanLogicalSwitchPortMigrateOptions mocks base method. -func (m *MockNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string) error { +func (m *MockNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string, fail bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CleanLogicalSwitchPortMigrateOptions", lspName) + ret := m.ctrl.Call(m, "CleanLogicalSwitchPortMigrateOptions", lspName, fail) ret0, _ := ret[0].(error) return ret0 } // CleanLogicalSwitchPortMigrateOptions indicates an expected call of CleanLogicalSwitchPortMigrateOptions. -func (mr *MockNbClientMockRecorder) CleanLogicalSwitchPortMigrateOptions(lspName any) *gomock.Call { +func (mr *MockNbClientMockRecorder) CleanLogicalSwitchPortMigrateOptions(lspName, fail any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockNbClient)(nil).CleanLogicalSwitchPortMigrateOptions), lspName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockNbClient)(nil).CleanLogicalSwitchPortMigrateOptions), lspName, fail) } // ClearLogicalRouterPolicy mocks base method. diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index b8b745f0304..1ae2ee12ba7 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -642,11 +642,11 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca // todo: isVmPod, getPodType, getNameByPod has duplicated logic var err error - var isMigrating bool + var isMigrate, migrated, fail bool var vmKey, srcNodeName, targetNodeName string if isVMPod && c.config.EnableKeepVMIP { vmKey = fmt.Sprintf("%s/%s", namespace, vmName) - if isMigrating, srcNodeName, targetNodeName, err = c.migrateVM(pod, vmKey); err != nil { + if isMigrate, migrated, fail, srcNodeName, targetNodeName, err = c.migrateVM(pod, vmKey); err != nil { klog.Error(err) return nil, err } @@ -738,20 +738,16 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca return nil, err } - if vmKey != "" { - if isMigrating { - if err = c.setVMLSPMigrationOptions(portName, srcNodeName, targetNodeName); err != nil { + if vmKey != "" && isMigrate { + if migrated { + if err = c.cleanVMLSPMigrationOptions(portName, fail); err != nil { klog.Error(err) return nil, err } - pod.Annotations[util.OVNMigrationAnnotation] = "true" } else { - if migrate, ok := pod.Annotations[util.OVNMigrationAnnotation]; ok && migrate == "true" { - if err = c.cleanVMLSPMigrationOptions(portName); err != nil { - klog.Error(err) - return nil, err - } - pod.Annotations[util.OVNMigratedAnnotation] = "true" + if err = c.setVMLSPMigrationOptions(portName, srcNodeName, targetNodeName); err != nil { + klog.Error(err) + return nil, err } } } @@ -2120,53 +2116,56 @@ func (c *Controller) getVirtualIPs(pod *v1.Pod, podNets []*kubeovnNet) map[strin return vipsMap } -// migrate vm return migrate, src node, target node, err -func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, string, string, error) { - if migrated, ok := pod.Annotations[util.OVNMigratedAnnotation]; ok && migrated == "true" { - klog.Infof("vm %s is migrated", vmKey) - return false, "", "", nil +// migrate vm return migrate, migrated, fail, src node, target node, err +func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, bool, bool, string, string, error) { + // try optimize vm migration, no need return error + // migrate true means need ovn set migrate options + // migrated ok means need set migrate options to target node + // migrated failed means need set migrate options to source node + isTarget, ok := pod.Annotations[util.MigrationTargetAnnotation] + // only handle target vm pod + if !ok || isTarget != "true" { + return false, false, false, "", "", nil + } + srcNode, ok := pod.Annotations[util.MigrationSourceNodeAnnotation] + if !ok || srcNode == "" { + err := fmt.Errorf("vm %s migration source node is not set", vmKey) + klog.Warning(err) + return false, false, false, "", "", nil + } + targetNode := pod.Spec.NodeName + if targetNode == "" { + err := fmt.Errorf("vm %s migration target node is not set", vmKey) + klog.Warning(err) + return false, false, false, "", "", nil } - migratePhase, ok := pod.Annotations[util.MigrationPhaseAnnotation] if !ok { - // not migrate vm - return false, "", "", nil + err := fmt.Errorf("vm %s migration phase is not set", vmKey) + klog.Warning(err) + return false, false, false, "", "", nil } // check migrate phase if migratePhase == "" { err := fmt.Errorf("vm %s migration phase is empty", vmKey) - klog.Error(err) - return false, "", "", err + klog.Warning(err) + return false, false, false, "", "", nil + } + if migratePhase == util.MigrationStateStarted { + klog.Infof("start to migrate src vm %s from %s to %s", vmKey, srcNode, targetNode) + return true, false, false, srcNode, targetNode, nil } if migratePhase == util.MigrationStateSucceeded { - klog.Infof("vm %s migration is succeeded", vmKey) - return false, "", "", nil + klog.Infof("manage to migrate src vm %s from %s to %s", vmKey, srcNode, targetNode) + + return true, true, false, srcNode, targetNode, nil } if migratePhase == util.MigrationStateFailed { - klog.Warningf("vm %s migration is failed", vmKey) - return false, "", "", nil + klog.Infof("failed to migrate src vm %s from %s to %s", vmKey, srcNode, targetNode) + return true, true, true, srcNode, targetNode, nil } - if migratePhase == util.MigrationStateStarted { - if isTarget, ok := pod.Annotations[util.MigrationTargetAnnotation]; ok && isTarget == "true" { - klog.Infof("start to migrate target vm %s", vmKey) - // this pod is target vm pod - srcNode, ok := pod.Annotations[util.MigrationSourceNodeAnnotation] - if !ok || srcNode == "" { - err := fmt.Errorf("vm %s migration source node is not set", vmKey) - klog.Error(err) - return false, "", "", err - } - targetNode := pod.Spec.NodeName - if targetNode == "" { - err := fmt.Errorf("vm %s migration target node is not set", vmKey) - klog.Error(err) - return false, "", "", err - } - return true, srcNode, targetNode, nil - } - klog.Infof("start to migrate src vm %s", vmKey) - } - return false, "", "", nil + + return false, false, false, "", "", nil } func (c *Controller) setVMLSPMigrationOptions(portName, srcNodeName, targetNodeName string) error { @@ -2179,9 +2178,9 @@ func (c *Controller) setVMLSPMigrationOptions(portName, srcNodeName, targetNodeN return nil } -func (c *Controller) cleanVMLSPMigrationOptions(portName string) error { +func (c *Controller) cleanVMLSPMigrationOptions(portName string, fail bool) error { klog.Infof("clean migrate options for lsp %s", portName) - if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(portName); err != nil { + if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(portName, fail); err != nil { err = fmt.Errorf("failed to clean migrate options for lsp %s, %v", portName, err) klog.Error(err) return err diff --git a/pkg/ovs/interface.go b/pkg/ovs/interface.go index 10c1e41ef65..ccf1f1fce0d 100644 --- a/pkg/ovs/interface.go +++ b/pkg/ovs/interface.go @@ -93,7 +93,7 @@ type LogicalSwitchPort interface { LogicalSwitchPortExists(name string) (bool, error) // vm live migrate SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string) error - CleanLogicalSwitchPortMigrateOptions(lspName string) error + CleanLogicalSwitchPortMigrateOptions(lspName string, fail bool) error } type LoadBalancer interface { diff --git a/pkg/ovs/ovn-nb-logical_switch_port.go b/pkg/ovs/ovn-nb-logical_switch_port.go index 0aeed7a76e8..f7309ea542a 100644 --- a/pkg/ovs/ovn-nb-logical_switch_port.go +++ b/pkg/ovs/ovn-nb-logical_switch_port.go @@ -863,7 +863,7 @@ func (c *OVNNbClient) GetLogicalSwitchPortMigrateOptions(lspName string) (*ovnnb } // CleanLogicalSwitchPortMigrateOptions clean logical switch port options of migration -func (c *OVNNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string) error { +func (c *OVNNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string, fail bool) error { lsp, src, target, err := c.GetLogicalSwitchPortMigrateOptions(lspName) if err != nil { klog.Error(err) @@ -871,8 +871,15 @@ func (c *OVNNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string) error } if src != "" && target != "" { lsp.Options = make(map[string]string) - lsp.Options["requested-chassis"] = target - klog.Infof("clean logical switch port %s options", lspName) + if fail { + // rollback + klog.Infof("clean migrator target options for port %s", lspName) + lsp.Options["requested-chassis"] = src + } else { + klog.Infof("clean migrator source options for port %s", lspName) + lsp.Options["requested-chassis"] = target + + } if err := c.UpdateLogicalSwitchPort(lsp, &lsp.Options); err != nil { err = fmt.Errorf("failed to clean options for logical switch port %s : %v", lspName, err) klog.Error(err) diff --git a/pkg/util/const.go b/pkg/util/const.go index 1213f934194..0a602ca76ea 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -293,7 +293,6 @@ const ( VhostUserSocketVolumeName = "vhostuser-sockets" OVNMigrationAnnotation = "ovn.kubernetes.io/migrate" // to control if ovn set lsp migrate options - OVNMigratedAnnotation = "ovn.kubernetes.io/migrated" // to control if ovn clean lsp migrate options MigrationSourceNodeAnnotation = "kubevirt.io/migration-source-node" // target pod has source node name MigrationSourceAnnotation = "kubevirt.io/migration-source" // migration source vm: true or false MigrationTargetAnnotation = "kubevirt.io/migration-target" // migration target vm: true or false From e929d0ff262a31227aafd64ccb94e46554922bdb Mon Sep 17 00:00:00 2001 From: bobz965 Date: Thu, 29 Feb 2024 15:05:22 +0800 Subject: [PATCH 04/12] check source vm Signed-off-by: bobz965 --- pkg/controller/init.go | 6 +++--- pkg/controller/pod.go | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/controller/init.go b/pkg/controller/init.go index 9de515a6a9c..61a24f6e72f 100644 --- a/pkg/controller/init.go +++ b/pkg/controller/init.go @@ -859,11 +859,11 @@ func updateFinalizers(c client.Client, list client.ObjectList, getObjectItem fun controllerutil.RemoveFinalizer(patchedObj, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(patchedObj, util.KubeOVNControllerFinalizer) - if err := c.Patch(context.Background(), patchedObj, client.MergeFrom(cachedObj)); err != nil && !k8serrors.IsNotFound(err) { - klog.Errorf("failed to sync finalizers for %s %s: %v", + if err := c.Patch(context.Background(), patchedObj, client.MergeFrom(cachedObj)); err != nil { + klog.Warningf("failed to sync finalizers for %s %s: %v", patchedObj.GetObjectKind().GroupVersionKind().Kind, cache.MetaObjectToName(patchedObj), err) - return err + i++ } i++ } diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 1ae2ee12ba7..2bc1458c996 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -2122,9 +2122,12 @@ func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, bool, bool, str // migrate true means need ovn set migrate options // migrated ok means need set migrate options to target node // migrated failed means need set migrate options to source node - isTarget, ok := pod.Annotations[util.MigrationTargetAnnotation] - // only handle target vm pod - if !ok || isTarget != "true" { + if _, ok := pod.Annotations[util.MigrationTargetAnnotation]; ok { + klog.Infof("prepare to migrate vm %s out from source node %s", vmKey, pod.Spec.NodeName) + return false, false, false, "", "", nil + } + // ovn set migrator only in the process of target vm pod + if _, ok := pod.Annotations[util.MigrationTargetAnnotation]; !ok { return false, false, false, "", "", nil } srcNode, ok := pod.Annotations[util.MigrationSourceNodeAnnotation] From 41d4ebaca1dd9d3e3cde612d6eb767c0ca3d250f Mon Sep 17 00:00:00 2001 From: bobz965 Date: Thu, 29 Feb 2024 15:34:12 +0800 Subject: [PATCH 05/12] check source vm node name Signed-off-by: bobz965 --- pkg/controller/pod.go | 20 ++++++++++++++++---- pkg/util/const.go | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 2bc1458c996..87fce4057c8 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -611,7 +611,6 @@ func (c *Controller) handleAddOrUpdatePod(key string) (err error) { return nil } pod = cachedPod.DeepCopy() - // check if allocate subnet is need. also allocate subnet when hotplug nic needAllocatePodNets := needAllocateSubnets(pod, podNets) if len(needAllocatePodNets) != 0 { if cachedPod, err = c.reconcileAllocateSubnets(cachedPod, pod, needAllocatePodNets); err != nil { @@ -1367,6 +1366,9 @@ func getNextHopByTunnelIP(gw []net.IP) string { } func needAllocateSubnets(pod *v1.Pod, nets []*kubeovnNet) []*kubeovnNet { + // check if allocate from subnet is need. + // allocate subnet when change subnet to hotplug nic + // allocate subnet when migrate vm if !isPodAlive(pod) { return nil } @@ -1375,9 +1377,14 @@ func needAllocateSubnets(pod *v1.Pod, nets []*kubeovnNet) []*kubeovnNet { return nets } + migrate := false + if _, ok := pod.Annotations[util.MigrationJobAnnotation]; ok { + migrate = true + } + result := make([]*kubeovnNet, 0, len(nets)) for _, n := range nets { - if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, n.ProviderName)] != "true" { + if migrate || pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, n.ProviderName)] != "true" { result = append(result, n) } } @@ -2122,8 +2129,13 @@ func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, bool, bool, str // migrate true means need ovn set migrate options // migrated ok means need set migrate options to target node // migrated failed means need set migrate options to source node - if _, ok := pod.Annotations[util.MigrationTargetAnnotation]; ok { - klog.Infof("prepare to migrate vm %s out from source node %s", vmKey, pod.Spec.NodeName) + if _, ok := pod.Annotations[util.MigrationSourceAnnotation]; ok { + if pod.Spec.NodeName == "" { + err := fmt.Errorf("source vm %s running pod %s should have node name", vmKey, pod.Name) + klog.Warning(err) + return false, false, false, "", "", nil + } + klog.Infof("prepare to migrate vm %s pod %s out from source node %s", vmKey, pod.Name, pod.Spec.NodeName) return false, false, false, "", "", nil } // ovn set migrator only in the process of target vm pod diff --git a/pkg/util/const.go b/pkg/util/const.go index 0a602ca76ea..3e04657b74d 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -293,6 +293,7 @@ const ( VhostUserSocketVolumeName = "vhostuser-sockets" OVNMigrationAnnotation = "ovn.kubernetes.io/migrate" // to control if ovn set lsp migrate options + MigrationJobAnnotation = "kubevirt.io/migrationJobName" // migration job name MigrationSourceNodeAnnotation = "kubevirt.io/migration-source-node" // target pod has source node name MigrationSourceAnnotation = "kubevirt.io/migration-source" // migration source vm: true or false MigrationTargetAnnotation = "kubevirt.io/migration-target" // migration target vm: true or false From e6431e464f1e9f55936855b0276c8e2c2722b111 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Thu, 29 Feb 2024 17:15:41 +0800 Subject: [PATCH 06/12] fix migrator option Signed-off-by: bobz965 --- pkg/controller/pod.go | 3 ++- pkg/ovs/ovn-nb-logical_switch_port.go | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 87fce4057c8..d7d852d3f0a 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -1378,7 +1378,8 @@ func needAllocateSubnets(pod *v1.Pod, nets []*kubeovnNet) []*kubeovnNet { } migrate := false - if _, ok := pod.Annotations[util.MigrationJobAnnotation]; ok { + if job, ok := pod.Annotations[util.MigrationJobAnnotation]; ok { + klog.Infof("migrate job %s for pod %s/%s", job, pod.Namespace, pod.Name) migrate = true } diff --git a/pkg/ovs/ovn-nb-logical_switch_port.go b/pkg/ovs/ovn-nb-logical_switch_port.go index f7309ea542a..059c231805b 100644 --- a/pkg/ovs/ovn-nb-logical_switch_port.go +++ b/pkg/ovs/ovn-nb-logical_switch_port.go @@ -870,16 +870,15 @@ func (c *OVNNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string, fail return err } if src != "" && target != "" { - lsp.Options = make(map[string]string) if fail { // rollback - klog.Infof("clean migrator target options for port %s", lspName) + klog.Infof("rollback fail migrator port %s", lspName) lsp.Options["requested-chassis"] = src } else { - klog.Infof("clean migrator source options for port %s", lspName) + klog.Infof("set target migrator port %s", lspName) lsp.Options["requested-chassis"] = target - } + delete(lsp.Options, "activation-strategy") if err := c.UpdateLogicalSwitchPort(lsp, &lsp.Options); err != nil { err = fmt.Errorf("failed to clean options for logical switch port %s : %v", lspName, err) klog.Error(err) From 8d8a6df85e841a37f193589e58965dadd93db920 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Thu, 29 Feb 2024 17:55:06 +0800 Subject: [PATCH 07/12] fix: remove no use code Signed-off-by: bobz965 --- pkg/controller/node.go | 4 +--- pkg/ovs/ovn-nb-load_balancer.go | 9 +++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pkg/controller/node.go b/pkg/controller/node.go index 8d35152e5d2..d226d304c30 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -585,9 +585,7 @@ func (c *Controller) updateProviderNetworkForNodeDeletion(pn *kubeovnv1.Provider } } if changed { - if newPn == nil { - newPn = pn.DeepCopy() - } + newPn = pn.DeepCopy() newPn.Spec.CustomInterfaces = customInterfaces } if newPn != nil { diff --git a/pkg/ovs/ovn-nb-load_balancer.go b/pkg/ovs/ovn-nb-load_balancer.go index 99ffd91cb50..bb8b67870e2 100644 --- a/pkg/ovs/ovn-nb-load_balancer.go +++ b/pkg/ovs/ovn-nb-load_balancer.go @@ -152,12 +152,9 @@ func (c *OVNNbClient) LoadBalancerDeleteVip(lbName, vipEndpoint string, ignoreHe klog.Errorf("failed to delete lb ip port mapping: %v", err) return err } - - if lbhc != nil { - if err = c.LoadBalancerDeleteHealthCheck(lbName, lbhc.UUID); err != nil { - klog.Errorf("failed to delete lb health check: %v", err) - return err - } + if err = c.LoadBalancerDeleteHealthCheck(lbName, lbhc.UUID); err != nil { + klog.Errorf("failed to delete lb health check: %v", err) + return err } } if lb == nil || len(lb.Vips) == 0 { From de8469d9b4feb955e6a676423e035712ea02ff09 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Fri, 1 Mar 2024 19:01:33 +0800 Subject: [PATCH 08/12] fix migrated pod force deleted Signed-off-by: bobz965 --- mocks/pkg/ovs/interface.go | 44 ++++++++++--- pkg/controller/pod.go | 58 +++++++---------- pkg/ovs/interface.go | 3 +- pkg/ovs/ovn-nb-logical_switch_port.go | 93 ++++++++++++++++++++------- 4 files changed, 130 insertions(+), 68 deletions(-) diff --git a/mocks/pkg/ovs/interface.go b/mocks/pkg/ovs/interface.go index 1aee8f0640a..87a9a645391 100644 --- a/mocks/pkg/ovs/interface.go +++ b/mocks/pkg/ovs/interface.go @@ -768,17 +768,17 @@ func (m *MockLogicalSwitchPort) EXPECT() *MockLogicalSwitchPortMockRecorder { } // CleanLogicalSwitchPortMigrateOptions mocks base method. -func (m *MockLogicalSwitchPort) CleanLogicalSwitchPortMigrateOptions(lspName string, fail bool) error { +func (m *MockLogicalSwitchPort) CleanLogicalSwitchPortMigrateOptions(lspName string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CleanLogicalSwitchPortMigrateOptions", lspName, fail) + ret := m.ctrl.Call(m, "CleanLogicalSwitchPortMigrateOptions", lspName) ret0, _ := ret[0].(error) return ret0 } // CleanLogicalSwitchPortMigrateOptions indicates an expected call of CleanLogicalSwitchPortMigrateOptions. -func (mr *MockLogicalSwitchPortMockRecorder) CleanLogicalSwitchPortMigrateOptions(lspName, fail any) *gomock.Call { +func (mr *MockLogicalSwitchPortMockRecorder) CleanLogicalSwitchPortMigrateOptions(lspName any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockLogicalSwitchPort)(nil).CleanLogicalSwitchPortMigrateOptions), lspName, fail) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockLogicalSwitchPort)(nil).CleanLogicalSwitchPortMigrateOptions), lspName) } // CreateBareLogicalSwitchPort mocks base method. @@ -973,6 +973,20 @@ func (mr *MockLogicalSwitchPortMockRecorder) LogicalSwitchPortExists(name any) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogicalSwitchPortExists", reflect.TypeOf((*MockLogicalSwitchPort)(nil).LogicalSwitchPortExists), name) } +// ResetLogicalSwitchPortMigrateOptions mocks base method. +func (m *MockLogicalSwitchPort) ResetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string, migratedFail bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResetLogicalSwitchPortMigrateOptions", lspName, srcNodeName, targetNodeName, migratedFail) + ret0, _ := ret[0].(error) + return ret0 +} + +// ResetLogicalSwitchPortMigrateOptions indicates an expected call of ResetLogicalSwitchPortMigrateOptions. +func (mr *MockLogicalSwitchPortMockRecorder) ResetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName, migratedFail any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockLogicalSwitchPort)(nil).ResetLogicalSwitchPortMigrateOptions), lspName, srcNodeName, targetNodeName, migratedFail) +} + // SetLogicalSwitchPortArpProxy mocks base method. func (m *MockLogicalSwitchPort) SetLogicalSwitchPortArpProxy(lspName string, enableArpProxy bool) error { m.ctrl.T.Helper() @@ -2463,17 +2477,17 @@ func (mr *MockNbClientMockRecorder) AddressSetUpdateAddress(asName any, addresse } // CleanLogicalSwitchPortMigrateOptions mocks base method. -func (m *MockNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string, fail bool) error { +func (m *MockNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CleanLogicalSwitchPortMigrateOptions", lspName, fail) + ret := m.ctrl.Call(m, "CleanLogicalSwitchPortMigrateOptions", lspName) ret0, _ := ret[0].(error) return ret0 } // CleanLogicalSwitchPortMigrateOptions indicates an expected call of CleanLogicalSwitchPortMigrateOptions. -func (mr *MockNbClientMockRecorder) CleanLogicalSwitchPortMigrateOptions(lspName, fail any) *gomock.Call { +func (mr *MockNbClientMockRecorder) CleanLogicalSwitchPortMigrateOptions(lspName any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockNbClient)(nil).CleanLogicalSwitchPortMigrateOptions), lspName, fail) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockNbClient)(nil).CleanLogicalSwitchPortMigrateOptions), lspName) } // ClearLogicalRouterPolicy mocks base method. @@ -4030,6 +4044,20 @@ func (mr *MockNbClientMockRecorder) RemoveLogicalPatchPort(lspName, lrpName any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveLogicalPatchPort", reflect.TypeOf((*MockNbClient)(nil).RemoveLogicalPatchPort), lspName, lrpName) } +// ResetLogicalSwitchPortMigrateOptions mocks base method. +func (m *MockNbClient) ResetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string, migratedFail bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResetLogicalSwitchPortMigrateOptions", lspName, srcNodeName, targetNodeName, migratedFail) + ret0, _ := ret[0].(error) + return ret0 +} + +// ResetLogicalSwitchPortMigrateOptions indicates an expected call of ResetLogicalSwitchPortMigrateOptions. +func (mr *MockNbClientMockRecorder) ResetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName, migratedFail any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetLogicalSwitchPortMigrateOptions", reflect.TypeOf((*MockNbClient)(nil).ResetLogicalSwitchPortMigrateOptions), lspName, srcNodeName, targetNodeName, migratedFail) +} + // SetACLLog mocks base method. func (m *MockNbClient) SetACLLog(pgName, protocol string, logEnable, isIngress bool) error { m.ctrl.T.Helper() diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index d7d852d3f0a..9f7957374dc 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -641,11 +641,11 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca // todo: isVmPod, getPodType, getNameByPod has duplicated logic var err error - var isMigrate, migrated, fail bool + var isMigrate, migrated, migratedFail bool var vmKey, srcNodeName, targetNodeName string if isVMPod && c.config.EnableKeepVMIP { vmKey = fmt.Sprintf("%s/%s", namespace, vmName) - if isMigrate, migrated, fail, srcNodeName, targetNodeName, err = c.migrateVM(pod, vmKey); err != nil { + if isMigrate, migrated, migratedFail, srcNodeName, targetNodeName, err = c.migrateVM(pod, vmKey); err != nil { klog.Error(err) return nil, err } @@ -729,7 +729,13 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca DHCPv4OptionsUUID: subnet.Status.DHCPv4OptionsUUID, DHCPv6OptionsUUID: subnet.Status.DHCPv6OptionsUUID, } - + if isVMPod && !isMigrate { + if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(portName); err != nil { + err = fmt.Errorf("failed to clean migrate options for lsp %s, %v", portName, err) + klog.Error(err) + return nil, err + } + } if err := c.OVNNbClient.CreateLogicalSwitchPort(subnet.Name, portName, ipStr, mac, podName, pod.Namespace, portSecurity, securityGroupAnnotation, vips, podNet.Subnet.Spec.EnableDHCP, dhcpOptions, subnet.Spec.Vpc); err != nil { c.recorder.Eventf(pod, v1.EventTypeWarning, "CreateOVNPortFailed", err.Error()) @@ -737,14 +743,18 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca return nil, err } - if vmKey != "" && isMigrate { + if isMigrate { if migrated { - if err = c.cleanVMLSPMigrationOptions(portName, fail); err != nil { + klog.Infof("migrate end reset options for lsp %s from %s to %s, migrated fail: %t", portName, srcNodeName, targetNodeName, migratedFail) + if err := c.OVNNbClient.ResetLogicalSwitchPortMigrateOptions(portName, srcNodeName, targetNodeName, migratedFail); err != nil { + err = fmt.Errorf("failed to clean migrate options for lsp %s, %v", portName, err) klog.Error(err) return nil, err } } else { - if err = c.setVMLSPMigrationOptions(portName, srcNodeName, targetNodeName); err != nil { + klog.Infof("migrate start set options for lsp %s from %s to %s", portName, srcNodeName, targetNodeName) + if err := c.OVNNbClient.SetLogicalSwitchPortMigrateOptions(portName, srcNodeName, targetNodeName); err != nil { + err = fmt.Errorf("failed to set migrate options for lsp %s, %v", portName, err) klog.Error(err) return nil, err } @@ -1379,7 +1389,7 @@ func needAllocateSubnets(pod *v1.Pod, nets []*kubeovnNet) []*kubeovnNet { migrate := false if job, ok := pod.Annotations[util.MigrationJobAnnotation]; ok { - klog.Infof("migrate job %s for pod %s/%s", job, pod.Namespace, pod.Name) + klog.Infof("pod %s/%s is in the migration job %s", pod.Namespace, pod.Name, job) migrate = true } @@ -2130,13 +2140,12 @@ func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, bool, bool, str // migrate true means need ovn set migrate options // migrated ok means need set migrate options to target node // migrated failed means need set migrate options to source node + if _, ok := pod.Annotations[util.MigrationJobAnnotation]; !ok { + return false, false, false, "", "", nil + } if _, ok := pod.Annotations[util.MigrationSourceAnnotation]; ok { - if pod.Spec.NodeName == "" { - err := fmt.Errorf("source vm %s running pod %s should have node name", vmKey, pod.Name) - klog.Warning(err) - return false, false, false, "", "", nil - } - klog.Infof("prepare to migrate vm %s pod %s out from source node %s", vmKey, pod.Name, pod.Spec.NodeName) + klog.Infof("will migrate out vm %s pod %s from source node %s", vmKey, pod.Name, pod.Spec.NodeName) + // 这里可能还有一个状态能标志结束，源pod更新了一次 return false, false, false, "", "", nil } // ovn set migrator only in the process of target vm pod @@ -2172,8 +2181,7 @@ func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, bool, bool, str return true, false, false, srcNode, targetNode, nil } if migratePhase == util.MigrationStateSucceeded { - klog.Infof("manage to migrate src vm %s from %s to %s", vmKey, srcNode, targetNode) - + klog.Infof("succeed to migrate src vm %s from %s to %s", vmKey, srcNode, targetNode) return true, true, false, srcNode, targetNode, nil } if migratePhase == util.MigrationStateFailed { @@ -2183,23 +2191,3 @@ func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, bool, bool, str return false, false, false, "", "", nil } - -func (c *Controller) setVMLSPMigrationOptions(portName, srcNodeName, targetNodeName string) error { - klog.Infof("set migrate options for lsp %s", portName) - if err := c.OVNNbClient.SetLogicalSwitchPortMigrateOptions(portName, srcNodeName, targetNodeName); err != nil { - err = fmt.Errorf("failed to set migrate options for lsp %s, %v", portName, err) - klog.Error(err) - return err - } - return nil -} - -func (c *Controller) cleanVMLSPMigrationOptions(portName string, fail bool) error { - klog.Infof("clean migrate options for lsp %s", portName) - if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(portName, fail); err != nil { - err = fmt.Errorf("failed to clean migrate options for lsp %s, %v", portName, err) - klog.Error(err) - return err - } - return nil -} diff --git a/pkg/ovs/interface.go b/pkg/ovs/interface.go index ccf1f1fce0d..cf084653e3c 100644 --- a/pkg/ovs/interface.go +++ b/pkg/ovs/interface.go @@ -93,7 +93,8 @@ type LogicalSwitchPort interface { LogicalSwitchPortExists(name string) (bool, error) // vm live migrate SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string) error - CleanLogicalSwitchPortMigrateOptions(lspName string, fail bool) error + ResetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string, migratedFail bool) error + CleanLogicalSwitchPortMigrateOptions(lspName string) error } type LoadBalancer interface { diff --git a/pkg/ovs/ovn-nb-logical_switch_port.go b/pkg/ovs/ovn-nb-logical_switch_port.go index 059c231805b..b4f4c31c69b 100644 --- a/pkg/ovs/ovn-nb-logical_switch_port.go +++ b/pkg/ovs/ovn-nb-logical_switch_port.go @@ -551,7 +551,9 @@ func (c *OVNNbClient) SetLogicalSwitchPortVlanTag(lspName string, vlanID int) er // UpdateLogicalSwitchPort update logical switch port func (c *OVNNbClient) UpdateLogicalSwitchPort(lsp *ovnnb.LogicalSwitchPort, fields ...interface{}) error { if lsp == nil { - return fmt.Errorf("logical_switch_port is nil") + err := fmt.Errorf("logical switch port is nil") + klog.Error(err) + return err } op, err := c.Where(lsp).Update(lsp, fields...) @@ -561,6 +563,7 @@ func (c *OVNNbClient) UpdateLogicalSwitchPort(lsp *ovnnb.LogicalSwitchPort, fiel } if err = c.Transact("lsp-update", op); err != nil { + klog.Error(err) return fmt.Errorf("update logical switch port %s: %v", lsp.Name, err) } @@ -820,20 +823,26 @@ func (c *OVNNbClient) SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, t klog.Error(err) return err } + if lsp == nil { + err := fmt.Errorf("no migrator logical switch port %s", lspName) + klog.Error(err) + return err + } + if src == srcNodeName && target == targetNodeName { // already set return nil } requestedChassis := fmt.Sprintf("%s,%s", srcNodeName, targetNodeName) - klog.Infof("set logical switch port %s options requested-chassis=%s", lspName, requestedChassis) if lsp.Options == nil { lsp.Options = make(map[string]string) } lsp.Options["requested-chassis"] = requestedChassis lsp.Options["activation-strategy"] = "rarp" + klog.Infof("set migrator logical switch port %s options: %v", lspName, lsp.Options) if err := c.UpdateLogicalSwitchPort(lsp, &lsp.Options); err != nil { - err = fmt.Errorf("failed to set logical switch port %s options requested chassis %s: %v", lspName, requestedChassis, err) + err = fmt.Errorf("failed to set migrator logical switch port %s options requested chassis %s: %v", lspName, requestedChassis, err) klog.Error(err) return err } @@ -842,14 +851,14 @@ func (c *OVNNbClient) SetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, t // GetLogicalSwitchPortMigrateOptions get logical switch port src and target node name options of migrate func (c *OVNNbClient) GetLogicalSwitchPortMigrateOptions(lspName string) (*ovnnb.LogicalSwitchPort, string, string, error) { - lsp, err := c.GetLogicalSwitchPort(lspName, true) + lsp, err := c.GetLogicalSwitchPort(lspName, false) if err != nil { - err = fmt.Errorf("failed to get logical switch port %s: %v", lspName, err) + err = fmt.Errorf("failed to get migrator logical switch port %s: %v", lspName, err) klog.Error(err) return nil, "", "", err } - if lsp == nil || lsp.Options == nil { - return nil, "", "", nil + if lsp.Options == nil { + return lsp, "", "", nil } requestedChassis, ok := lsp.Options["requested-chassis"] @@ -862,28 +871,64 @@ func (c *OVNNbClient) GetLogicalSwitchPortMigrateOptions(lspName string) (*ovnnb return nil, "", "", nil } +func (c *OVNNbClient) ResetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string, migratedFail bool) error { + lsp, err := c.GetLogicalSwitchPort(lspName, false) + if err != nil { + err = fmt.Errorf("failed to get migrator logical switch port %s: %v", lspName, err) + klog.Error(err) + return err + } + if lsp.Options == nil { + klog.Infof("logical switch port %s has no options", lspName) + return nil + } + if _, ok := lsp.Options["requested-chassis"]; !ok { + klog.Infof("logical switch port %s has no migrator options", lspName) + return nil + } + if migratedFail { + // rollback + klog.Infof("reset migrator port %s to source node %s", lspName, srcNodeName) + lsp.Options["requested-chassis"] = srcNodeName + } else { + klog.Infof("reset migrator port %s to target node %s", lspName, targetNodeName) + lsp.Options["requested-chassis"] = targetNodeName + } + delete(lsp.Options, "activation-strategy") + klog.Infof("reset migrator logical switch port %s options: %v", lspName, lsp.Options) + if err := c.UpdateLogicalSwitchPort(lsp, &lsp.Options); err != nil { + err = fmt.Errorf("failed to reset options for migrator logical switch port %s: %v", lspName, err) + klog.Error(err) + return err + } + return nil +} + // CleanLogicalSwitchPortMigrateOptions clean logical switch port options of migration -func (c *OVNNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string, fail bool) error { - lsp, src, target, err := c.GetLogicalSwitchPortMigrateOptions(lspName) +func (c *OVNNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string) error { + lsp, err := c.GetLogicalSwitchPort(lspName, true) if err != nil { + err = fmt.Errorf("failed to get migrator logical switch port %s: %v", lspName, err) klog.Error(err) return err } - if src != "" && target != "" { - if fail { - // rollback - klog.Infof("rollback fail migrator port %s", lspName) - lsp.Options["requested-chassis"] = src - } else { - klog.Infof("set target migrator port %s", lspName) - lsp.Options["requested-chassis"] = target - } - delete(lsp.Options, "activation-strategy") - if err := c.UpdateLogicalSwitchPort(lsp, &lsp.Options); err != nil { - err = fmt.Errorf("failed to clean options for logical switch port %s : %v", lspName, err) - klog.Error(err) - return err - } + if lsp == nil { + return nil + } + if lsp.Options == nil { + return nil + } + if _, ok := lsp.Options["requested-chassis"]; !ok { + return nil + } + // migrated pod port has requested-chassis, if pod force deleted, the vm pod may schedule to another node + // if not clean the requested-chassis, the pod has no network connectivity + delete(lsp.Options, "requested-chassis") + klog.Infof("cleaned migrator logical switch port %s options: %v", lspName, lsp.Options) + if err := c.UpdateLogicalSwitchPort(lsp, &lsp.Options); err != nil { + err = fmt.Errorf("failed to clean options for migrator logical switch port %s: %v", lspName, err) + klog.Error(err) + return err } return nil } From fa8b35f69313753c6a1eaa73e6b1011a5c3393a0 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Sat, 2 Mar 2024 14:06:27 +0800 Subject: [PATCH 09/12] fix: CleanLogicalSwitchPortMigrateOptions Signed-off-by: bobz965 --- pkg/controller/pod.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 9f7957374dc..62d3af110a2 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -729,13 +729,6 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca DHCPv4OptionsUUID: subnet.Status.DHCPv4OptionsUUID, DHCPv6OptionsUUID: subnet.Status.DHCPv6OptionsUUID, } - if isVMPod && !isMigrate { - if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(portName); err != nil { - err = fmt.Errorf("failed to clean migrate options for lsp %s, %v", portName, err) - klog.Error(err) - return nil, err - } - } if err := c.OVNNbClient.CreateLogicalSwitchPort(subnet.Name, portName, ipStr, mac, podName, pod.Namespace, portSecurity, securityGroupAnnotation, vips, podNet.Subnet.Spec.EnableDHCP, dhcpOptions, subnet.Spec.Vpc); err != nil { c.recorder.Eventf(pod, v1.EventTypeWarning, "CreateOVNPortFailed", err.Error()) @@ -1149,6 +1142,19 @@ func (c *Controller) handleDeletePod(key string) error { return err } } + } else if isVMPod { + ports, err := c.OVNNbClient.ListNormalLogicalSwitchPorts(true, map[string]string{"pod": podKey}) + if err != nil { + klog.Errorf("failed to list lsps of pod '%s', %v", pod.Name, err) + return err + } + for _, port := range ports { + if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(port.Name); err != nil { + err = fmt.Errorf("failed to clean migrate options for lsp %s, %v", port.Name, err) + klog.Error(err) + return err + } + } } for _, podNet := range podNets { c.syncVirtualPortsQueue.Add(podNet.Subnet.Name) From cb82dec4ace7d94f2cd552d0d189f7209eeaf578 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Sat, 2 Mar 2024 15:51:03 +0800 Subject: [PATCH 10/12] fix lost vendor Signed-off-by: bobz965 --- pkg/controller/pod.go | 26 +++++++++++++------------- pkg/ovs/ovn-nb-logical_switch_port.go | 5 ++--- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 62d3af110a2..519d46d472e 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -1036,6 +1036,19 @@ func (c *Controller) handleDeletePod(key string) error { } isVMPod, vmName := isVMPod(pod) if isVMPod && c.config.EnableKeepVMIP { + ports, err := c.OVNNbClient.ListNormalLogicalSwitchPorts(true, map[string]string{"pod": podKey}) + if err != nil { + klog.Errorf("failed to list lsps of pod '%s', %v", pod.Name, err) + return err + } + for _, port := range ports { + klog.Infof("clean migrate options for vm lsp %s", port.Name) + if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(port.Name); err != nil { + err = fmt.Errorf("failed to clean migrate options for vm lsp %s, %v", port.Name, err) + klog.Error(err) + return err + } + } vmToBeDel := c.isVMToDel(pod, vmName) isDelete, err := appendCheckPodToDel(c, pod, vmName, util.VMInstance) if pod.DeletionTimestamp != nil { @@ -1142,19 +1155,6 @@ func (c *Controller) handleDeletePod(key string) error { return err } } - } else if isVMPod { - ports, err := c.OVNNbClient.ListNormalLogicalSwitchPorts(true, map[string]string{"pod": podKey}) - if err != nil { - klog.Errorf("failed to list lsps of pod '%s', %v", pod.Name, err) - return err - } - for _, port := range ports { - if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(port.Name); err != nil { - err = fmt.Errorf("failed to clean migrate options for lsp %s, %v", port.Name, err) - klog.Error(err) - return err - } - } } for _, podNet := range podNets { c.syncVirtualPortsQueue.Add(podNet.Subnet.Name) diff --git a/pkg/ovs/ovn-nb-logical_switch_port.go b/pkg/ovs/ovn-nb-logical_switch_port.go index b4f4c31c69b..dc468d5f5d1 100644 --- a/pkg/ovs/ovn-nb-logical_switch_port.go +++ b/pkg/ovs/ovn-nb-logical_switch_port.go @@ -33,7 +33,7 @@ func buildLogicalSwitchPort(lspName, lsName, ip, mac, podName, namespace string, // addresses is the first element of addresses lsp.Addresses = []string{strings.TrimSpace(strings.Join(addresses, " "))} - + lsp.ExternalIDs["vendor"] = util.CniTypeName if portSecurity { if len(vips) != 0 { addresses = append(addresses, vipList...) @@ -868,7 +868,7 @@ func (c *OVNNbClient) GetLogicalSwitchPortMigrateOptions(lspName string) (*ovnnb return lsp, splits[0], splits[1], nil } } - return nil, "", "", nil + return lsp, "", "", nil } func (c *OVNNbClient) ResetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, targetNodeName string, migratedFail bool) error { @@ -895,7 +895,6 @@ func (c *OVNNbClient) ResetLogicalSwitchPortMigrateOptions(lspName, srcNodeName, lsp.Options["requested-chassis"] = targetNodeName } delete(lsp.Options, "activation-strategy") - klog.Infof("reset migrator logical switch port %s options: %v", lspName, lsp.Options) if err := c.UpdateLogicalSwitchPort(lsp, &lsp.Options); err != nil { err = fmt.Errorf("failed to reset options for migrator logical switch port %s: %v", lspName, err) klog.Error(err) From 6eb5deafb30d295dce38b12e1537e5810005cd8a Mon Sep 17 00:00:00 2001 From: bobz965 Date: Mon, 4 Mar 2024 16:56:41 +0800 Subject: [PATCH 11/12] Update pkg/util/const.go Co-authored-by: Oilbeater --- pkg/util/const.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/util/const.go b/pkg/util/const.go index 3e04657b74d..98d371bfefa 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -298,7 +298,7 @@ const ( MigrationSourceAnnotation = "kubevirt.io/migration-source" // migration source vm: true or false MigrationTargetAnnotation = "kubevirt.io/migration-target" // migration target vm: true or false MigrationPhaseAnnotation = "kubevirt.io/migration-phase" // migration vm phase: started/succeeded/failed - MigrationStateStarted = "started" - MigrationStateSucceeded = "succeeded" - MigrationStateFailed = "failed" + MigrationPhaseStarted = "started" + MigrationPhaseSucceeded = "succeeded" + MigrationPhaseFailed = "failed" ) From 887a8bc0654a20f0e5fa1ccb5b21f190cd2f6336 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Mon, 4 Mar 2024 18:04:41 +0800 Subject: [PATCH 12/12] fix as review Signed-off-by: bobz965 --- pkg/controller/init.go | 6 +++--- pkg/controller/pod.go | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/controller/init.go b/pkg/controller/init.go index 61a24f6e72f..9de515a6a9c 100644 --- a/pkg/controller/init.go +++ b/pkg/controller/init.go @@ -859,11 +859,11 @@ func updateFinalizers(c client.Client, list client.ObjectList, getObjectItem fun controllerutil.RemoveFinalizer(patchedObj, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(patchedObj, util.KubeOVNControllerFinalizer) - if err := c.Patch(context.Background(), patchedObj, client.MergeFrom(cachedObj)); err != nil { - klog.Warningf("failed to sync finalizers for %s %s: %v", + if err := c.Patch(context.Background(), patchedObj, client.MergeFrom(cachedObj)); err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to sync finalizers for %s %s: %v", patchedObj.GetObjectKind().GroupVersionKind().Kind, cache.MetaObjectToName(patchedObj), err) - i++ + return err } i++ } diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 519d46d472e..36a8ceb7ff7 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -2151,7 +2151,6 @@ func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, bool, bool, str } if _, ok := pod.Annotations[util.MigrationSourceAnnotation]; ok { klog.Infof("will migrate out vm %s pod %s from source node %s", vmKey, pod.Name, pod.Spec.NodeName) - // 这里可能还有一个状态能标志结束，源pod更新了一次 return false, false, false, "", "", nil } // ovn set migrator only in the process of target vm pod @@ -2182,15 +2181,15 @@ func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, bool, bool, str klog.Warning(err) return false, false, false, "", "", nil } - if migratePhase == util.MigrationStateStarted { + if migratePhase == util.MigrationPhaseStarted { klog.Infof("start to migrate src vm %s from %s to %s", vmKey, srcNode, targetNode) return true, false, false, srcNode, targetNode, nil } - if migratePhase == util.MigrationStateSucceeded { + if migratePhase == util.MigrationPhaseSucceeded { klog.Infof("succeed to migrate src vm %s from %s to %s", vmKey, srcNode, targetNode) return true, true, false, srcNode, targetNode, nil } - if migratePhase == util.MigrationStateFailed { + if migratePhase == util.MigrationPhaseFailed { klog.Infof("failed to migrate src vm %s from %s to %s", vmKey, srcNode, targetNode) return true, true, true, srcNode, targetNode, nil }