Skip to content

Commit 94644e1

Browse files
changluyichangluyi
authored
u2o support custom vpc release 1.11 (#2849)
1 parent 30f4cc3 commit 94644e1

File tree

5 files changed

+128
-24
lines changed

5 files changed

+128
-24
lines changed

‎dist/images/install.sh

+2
Original file line numberDiff line numberDiff line change
@@ -1370,6 +1370,8 @@ spec:
13701370
type: string
13711371
u2oInterconnectionIP:
13721372
type: string
1373+
u2oInterconnectionVPC:
1374+
type: string
13731375
conditions:
13741376
type: array
13751377
items:

‎kubeovn-helm/templates/kube-ovn-crd.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -1142,6 +1142,8 @@ spec:
11421142
type: string
11431143
u2oInterconnectionIP:
11441144
type: string
1145+
u2oInterconnectionVPC:
1146+
type: string
11451147
conditions:
11461148
type: array
11471149
items:

‎pkg/apis/kubeovn/v1/types.go

+9-8
Original file line numberDiff line numberDiff line change
@@ -179,14 +179,15 @@ type SubnetStatus struct {
179179
// +patchStrategy=merge
180180
Conditions []SubnetCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
181181

182-
V4AvailableIPs float64 `json:"v4availableIPs"`
183-
V4UsingIPs float64 `json:"v4usingIPs"`
184-
V6AvailableIPs float64 `json:"v6availableIPs"`
185-
V6UsingIPs float64 `json:"v6usingIPs"`
186-
ActivateGateway string `json:"activateGateway"`
187-
DHCPv4OptionsUUID string `json:"dhcpV4OptionsUUID"`
188-
DHCPv6OptionsUUID string `json:"dhcpV6OptionsUUID"`
189-
U2OInterconnectionIP string `json:"u2oInterconnectionIP"`
182+
V4AvailableIPs float64 `json:"v4availableIPs"`
183+
V4UsingIPs float64 `json:"v4usingIPs"`
184+
V6AvailableIPs float64 `json:"v6availableIPs"`
185+
V6UsingIPs float64 `json:"v6usingIPs"`
186+
ActivateGateway string `json:"activateGateway"`
187+
DHCPv4OptionsUUID string `json:"dhcpV4OptionsUUID"`
188+
DHCPv6OptionsUUID string `json:"dhcpV6OptionsUUID"`
189+
U2OInterconnectionIP string `json:"u2oInterconnectionIP"`
190+
U2OInterconnectionVPC string `json:"u2oInterconnectionVPC"`
190191
}
191192

192193
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

‎pkg/controller/subnet.go

+113-16
Original file line numberDiff line numberDiff line change
@@ -562,9 +562,11 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error {
562562
return err
563563
}
564564

565-
if err := c.reconcileU2OInterconnectionIP(subnet); err != nil {
566-
klog.Errorf("failed to reconcile underlay subnet %s to overlay interconnection %v", subnet.Name, err)
567-
return err
565+
if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway {
566+
if err := c.reconcileU2OInterconnectionIP(subnet); err != nil {
567+
klog.Errorf("failed to reconcile underlay subnet %s to overlay interconnection %v", subnet.Name, err)
568+
return err
569+
}
568570
}
569571

570572
if !isOvnSubnet(subnet) {
@@ -657,6 +659,12 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error {
657659
if subnet.Status.U2OInterconnectionIP != "" && subnet.Spec.U2OInterconnection {
658660
gateway = subnet.Status.U2OInterconnectionIP
659661
}
662+
663+
if err := c.clearOldU2OResource(subnet); err != nil {
664+
klog.Errorf("clear subnet %s old u2o resource failed: %v", subnet.Name, err)
665+
return err
666+
}
667+
660668
if err := c.ovnLegacyClient.SetLogicalSwitchConfig(subnet.Name, vpc.Status.Router, subnet.Spec.Protocol, subnet.Spec.CIDRBlock, gateway, subnet.Spec.ExcludeIps, needRouter); err != nil {
661669
c.patchSubnetStatus(subnet, "SetLogicalSwitchConfigFailed", err.Error())
662670
return err
@@ -720,6 +728,11 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error {
720728
return err
721729
}
722730

731+
subnet.Status.U2OInterconnectionVPC = ""
732+
if subnet.Spec.U2OInterconnection {
733+
subnet.Status.U2OInterconnectionVPC = vpc.Status.Router
734+
}
735+
723736
if subnet.Spec.Private {
724737
if err := c.ovnLegacyClient.SetPrivateLogicalSwitch(subnet.Name, subnet.Spec.CIDRBlock, subnet.Spec.AllowSubnets); err != nil {
725738
c.patchSubnetStatus(subnet, "SetPrivateLogicalSwitchFailed", err.Error())
@@ -842,6 +855,13 @@ func (c *Controller) handleDeleteSubnet(subnet *kubeovnv1.Subnet) error {
842855
}
843856
}
844857

858+
if subnet.Spec.Vpc != c.config.ClusterRouter {
859+
if err := c.deleteCustomVPCPolicyRoutesForSubnet(subnet); err != nil {
860+
klog.Errorf("failed to delete custom vpc routes subnet %s, %v", subnet.Name, err)
861+
return err
862+
}
863+
}
864+
845865
klog.Infof("delete policy route for %s subnet %s", subnet.Spec.GatewayType, subnet.Name)
846866
if err := c.deletePolicyRouteByGatewayType(subnet, subnet.Spec.GatewayType, true); err != nil {
847867
klog.Errorf("failed to delete policy route for overlay subnet %s, %v", subnet.Name, err)
@@ -915,6 +935,13 @@ func (c *Controller) reconcileSubnet(subnet *kubeovnv1.Subnet) error {
915935
return err
916936
}
917937

938+
if subnet.Spec.Vpc != c.config.ClusterRouter {
939+
if err := c.reconcileOvnCustomVpcRoute(subnet); err != nil {
940+
klog.Errorf("reconcile custom vpc ovn route for subnet %s failed: %v", subnet.Name, err)
941+
return err
942+
}
943+
}
944+
918945
if err := c.reconcileVlan(subnet); err != nil {
919946
klog.Errorf("reconcile vlan for subnet %s failed, %v", subnet.Name, err)
920947
return err
@@ -1434,6 +1461,21 @@ func (c *Controller) reconcileOvnRoute(subnet *kubeovnv1.Subnet) error {
14341461
return nil
14351462
}
14361463

1464+
func (c *Controller) reconcileOvnCustomVpcRoute(subnet *kubeovnv1.Subnet) error {
1465+
if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway && subnet.Spec.U2OInterconnection && subnet.Status.U2OInterconnectionIP != "" {
1466+
if err := c.addPolicyRouteForU2OInterconn(subnet); err != nil {
1467+
klog.Errorf("failed to add policy route for underlay to overlay subnet interconnection %s %v", subnet.Name, err)
1468+
return err
1469+
}
1470+
}
1471+
1472+
if err := c.addCustomVPCPolicyRoutesForSubnet(subnet); err != nil {
1473+
klog.Error(err)
1474+
return err
1475+
}
1476+
return nil
1477+
}
1478+
14371479
func (c *Controller) deleteStaticRoute(ip, router string) error {
14381480
for _, ipStr := range strings.Split(ip, ",") {
14391481
if err := c.ovnLegacyClient.DeleteStaticRoute(ipStr, router); err != nil {
@@ -1840,7 +1882,7 @@ func (c *Controller) addCommonRoutesForSubnet(subnet *kubeovnv1.Subnet) error {
18401882
if !exist {
18411883
externalIDs := map[string]string{"vendor": util.CniTypeName, "subnet": subnet.Name}
18421884
klog.Infof("add policy route for router: %s, match %s, action %s, nexthop %s, extrenalID %v", c.config.ClusterRouter, match, "allow", "", externalIDs)
1843-
if err = c.ovnLegacyClient.AddPolicyRoute(c.config.ClusterRouter, util.SubnetRouterPolicyPriority, match, "allow", "", externalIDs); err != nil {
1885+
if err = c.ovnLegacyClient.AddPolicyRoute(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match, "allow", "", externalIDs); err != nil {
18441886
klog.Errorf("failed to add logical router policy for CIDR %s of subnet %s: %v", cidr, subnet.Name, err)
18451887
return err
18461888
}
@@ -2146,7 +2188,7 @@ func (c *Controller) addPolicyRouteForU2OInterconn(subnet *kubeovnv1.Subnet) err
21462188
prio 31000 match: "ip4.dst == underlay subnet cidr && ip4.dst != node ips" action: allow
21472189
21482190
policy2:
2149-
prio 31000 match: "ip4.dst == node ips && ip4.src == underlay subnet cidr" action: reoute physical gw
2191+
prio 31000 match: "ip4.dst == node ips && ip4.src == underlay subnet cidr" action: reroute physical gw
21502192
21512193
policy3:
21522194
prio 29000 match: "ip4.src == underlay subnet cidr" action: reroute physical gw
@@ -2155,16 +2197,19 @@ func (c *Controller) addPolicyRouteForU2OInterconn(subnet *kubeovnv1.Subnet) err
21552197
policy1 and policy2 allow overlay pod access underlay but when overlay pod access node ip, it should go join subnet,
21562198
policy3: underlay pod first access u2o interconnection lrp and then reoute to physical gw
21572199
*/
2158-
klog.Infof("add u2o interconnection policy for router: %s, match %s, action %s", subnet.Spec.Vpc, match1, "allow")
2159-
if err := c.ovnLegacyClient.AddPolicyRoute(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match1, "allow", "", externalIDs); err != nil {
2160-
klog.Errorf("failed to add u2o interconnection policy1 for subnet %s %v", subnet.Name, err)
2161-
return err
2162-
}
21632200

2164-
klog.Infof("add u2o interconnection policy for router: %s, match %s, action %s, nexthop %s", subnet.Spec.Vpc, match2, "reroute", nextHop)
2165-
if err := c.ovnLegacyClient.AddPolicyRoute(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match2, "reroute", nextHop, externalIDs); err != nil {
2166-
klog.Errorf("failed to add u2o interconnection policy2 for subnet %s %v", subnet.Name, err)
2167-
return err
2201+
if subnet.Spec.Vpc == c.config.ClusterRouter {
2202+
klog.Infof("add u2o interconnection policy for router: %s, match %s, action %s", subnet.Spec.Vpc, match1, "allow")
2203+
if err := c.ovnLegacyClient.AddPolicyRoute(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match1, "allow", "", externalIDs); err != nil {
2204+
klog.Errorf("failed to add u2o interconnection policy1 for subnet %s %v", subnet.Name, err)
2205+
return err
2206+
}
2207+
2208+
klog.Infof("add u2o interconnection policy for router: %s, match %s, action %s, nexthop %s", subnet.Spec.Vpc, match2, "reroute", nextHop)
2209+
if err := c.ovnLegacyClient.AddPolicyRoute(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match2, "reroute", nextHop, externalIDs); err != nil {
2210+
klog.Errorf("failed to add u2o interconnection policy2 for subnet %s %v", subnet.Name, err)
2211+
return err
2212+
}
21682213
}
21692214

21702215
klog.Infof("add u2o interconnection policy for router: %s, match %s, action %s, nexthop %s", subnet.Spec.Vpc, match3, "reroute", nextHop)
@@ -2191,13 +2236,19 @@ func (c *Controller) deletePolicyRouteForU2OInterconn(subnet *kubeovnv1.Subnet)
21912236
return nil
21922237
}
21932238

2239+
lr := subnet.Status.U2OInterconnectionVPC
2240+
if lr == "" {
2241+
// old version field U2OInterconnectionVPC may be "" and then use subnet.Spec.Vpc
2242+
lr = subnet.Spec.Vpc
2243+
}
2244+
21942245
var uuids []string
21952246
for _, result := range results {
21962247
uuids = append(uuids, result["_uuid"][0])
2197-
klog.Infof("delete u2o interconnection policy for router %s with match %s priority %s ", subnet.Spec.Vpc, result["match"], result["priority"])
2248+
klog.Infof("delete u2o interconnection policy for router %s with match %s priority %s ", lr, result["match"], result["priority"])
21982249
}
21992250

2200-
if err := c.ovnLegacyClient.DeletePolicyRouteByUUID(subnet.Spec.Vpc, uuids); err != nil {
2251+
if err := c.ovnLegacyClient.DeletePolicyRouteByUUID(lr, uuids); err != nil {
22012252
klog.Errorf("failed to delete u2o interconnection policy for subnet %s: %v", subnet.Name, err)
22022253
return err
22032254
}
@@ -2217,3 +2268,49 @@ func (c *Controller) deletePolicyRouteForU2OInterconn(subnet *kubeovnv1.Subnet)
22172268

22182269
return nil
22192270
}
2271+
2272+
func (c *Controller) addCustomVPCPolicyRoutesForSubnet(subnet *kubeovnv1.Subnet) error {
2273+
return c.addCommonRoutesForSubnet(subnet)
2274+
}
2275+
2276+
func (c *Controller) deleteCustomVPCPolicyRoutesForSubnet(subnet *kubeovnv1.Subnet) error {
2277+
2278+
for _, cidr := range strings.Split(subnet.Spec.CIDRBlock, ",") {
2279+
af := 4
2280+
if util.CheckProtocol(cidr) == kubeovnv1.ProtocolIPv6 {
2281+
af = 6
2282+
}
2283+
match := fmt.Sprintf("ip%d.dst == %s", af, cidr)
2284+
klog.Infof("delete policy route for router: %s, priority: %d, match %s", subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match)
2285+
if err := c.ovnLegacyClient.DeletePolicyRoute(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match); err != nil {
2286+
klog.Errorf("failed to delete logical router policy for CIDR %s of subnet %s: %v", cidr, subnet.Name, err)
2287+
return err
2288+
}
2289+
}
2290+
return nil
2291+
}
2292+
2293+
func (c *Controller) clearOldU2OResource(subnet *kubeovnv1.Subnet) error {
2294+
if subnet.Status.U2OInterconnectionVPC != "" &&
2295+
(!subnet.Spec.U2OInterconnection || (subnet.Spec.U2OInterconnection && subnet.Status.U2OInterconnectionVPC != subnet.Spec.Vpc)) {
2296+
// remove old u2o lsp and lrp first
2297+
lspName := fmt.Sprintf("%s-%s", subnet.Name, subnet.Status.U2OInterconnectionVPC)
2298+
lrpName := fmt.Sprintf("%s-%s", subnet.Status.U2OInterconnectionVPC, subnet.Name)
2299+
klog.Infof("clean subnet %s old u2o resource with lsp %s lrp %s ", subnet.Name, lspName, lrpName)
2300+
if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(lspName); err != nil {
2301+
klog.Errorf("failed to delete u2o logical switch port %s: %v", lspName, err)
2302+
return err
2303+
}
2304+
2305+
if err := c.ovnLegacyClient.DeleteLogicalRouterPort(lrpName); err != nil {
2306+
klog.Errorf("failed to delete u2o logical router port %s: %v", lrpName, err)
2307+
return err
2308+
}
2309+
2310+
if err := c.deletePolicyRouteForU2OInterconn(subnet); err != nil {
2311+
klog.Errorf("failed to delete u2o policy route for u2o connection %s: %v", subnet.Name, err)
2312+
return err
2313+
}
2314+
}
2315+
return nil
2316+
}

‎yamls/crd.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -1150,6 +1150,8 @@ spec:
11501150
type: string
11511151
u2oInterconnectionIP:
11521152
type: string
1153+
u2oInterconnectionVPC:
1154+
type: string
11531155
conditions:
11541156
type: array
11551157
items:

0 commit comments

Comments
 (0)