Skip to content

Commit bb1b85e

Browse files
zhangzujianzhangzujian
committed
cni-server: clear iptables mark before doing masquerade
1 parent d043a2d commit bb1b85e

File tree

1 file changed

+58
-21
lines changed

1 file changed

+58
-21
lines changed

‎pkg/daemon/gateway_linux.go

+58-21
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ const (
4343
Postrouting = "POSTROUTING"
4444
OvnPrerouting = "OVN-PREROUTING"
4545
OvnPostrouting = "OVN-POSTROUTING"
46+
OvnNodePort = "OVN-NODE-PORT"
4647
OvnNatOutGoingPolicy = "OVN-NAT-POLICY"
4748
OvnNatOutGoingPolicySubnet = "OVN-NAT-PSUBNET-"
4849
)
@@ -420,18 +421,36 @@ func (c *Controller) createIptablesRule(ipt *iptables.IPTables, rule util.IPTabl
420421
return nil
421422
}
422423

423-
func (c *Controller) updateIptablesChain(ipt *iptables.IPTables, table, chain, parent string, rules []util.IPTableRule) error {
424+
// ensure the iptables chain exists
425+
func (c *Controller) createIptablesChain(ipt *iptables.IPTables, table, chain string) error {
426+
if chain == "" {
427+
return nil
428+
}
429+
424430
ok, err := ipt.ChainExists(table, chain)
425431
if err != nil {
426432
klog.Errorf("failed to check existence of iptables chain %s in table %s: %v", chain, table, err)
427433
return err
428434
}
429-
if !ok {
430-
if err = ipt.NewChain(table, chain); err != nil {
431-
klog.Errorf("failed to create iptables chain %s in table %s: %v", chain, table, err)
432-
return err
433-
}
434-
klog.Infof("created iptables chain %s in table %s", chain, table)
435+
if ok {
436+
return nil
437+
}
438+
439+
if err = ipt.NewChain(table, chain); err != nil {
440+
klog.Errorf("failed to create iptables chain %s in table %s: %v", chain, table, err)
441+
return err
442+
}
443+
444+
klog.Infof("created iptables chain %s in table %s", chain, table)
445+
return nil
446+
}
447+
448+
func (c *Controller) updateIptablesChain(ipt *iptables.IPTables, table, chain, parent string, rules []util.IPTableRule) error {
449+
if err := c.createIptablesChain(ipt, table, chain); err != nil {
450+
return err
451+
}
452+
if err := c.createIptablesChain(ipt, table, parent); err != nil {
453+
return err
435454
}
436455
if parent != "" {
437456
comment := fmt.Sprintf("kube-ovn %s rules", strings.ToLower(parent))
@@ -440,7 +459,7 @@ func (c *Controller) updateIptablesChain(ipt *iptables.IPTables, table, chain, p
440459
Chain: parent,
441460
Rule: []string{"-m", "comment", "--comment", comment, "-j", chain},
442461
}
443-
if err = c.createIptablesRule(ipt, rule); err != nil {
462+
if err := c.createIptablesRule(ipt, rule); err != nil {
444463
klog.Errorf("failed to create iptables rule: %v", err)
445464
return err
446465
}
@@ -512,14 +531,12 @@ func (c *Controller) setIptables() error {
512531
v4Rules = []util.IPTableRule{
513532
// mark packets from pod to service
514533
{Table: NAT, Chain: OvnPrerouting, Rule: strings.Fields(`-i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000`)},
515-
// nat packets marked by kube-proxy or kube-ovn
516-
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)},
534+
// packets marked by kube-proxy or kube-ovn will be SNATed later by kube-proxy
535+
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j RETURN`)},
536+
// handle node port with external traffic policy set to Local
537+
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j ` + OvnNodePort)},
517538
// nat service traffic
518539
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j MASQUERADE`)},
519-
// do not nat node port service traffic with external traffic policy set to local
520-
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN`)},
521-
// nat node port service traffic with external traffic policy set to local for subnets with centralized gateway
522-
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j MASQUERADE`)},
523540
// do not nat reply packets in direct routing
524541
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},
525542
// do not nat route traffic
@@ -531,6 +548,12 @@ func (c *Controller) setIptables() error {
531548
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j RETURN`, OnOutGoingForwardMark))},
532549
// default nat outgoing rules
533550
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`)},
551+
// clear mark
552+
{Table: NAT, Chain: OvnNodePort, Rule: strings.Fields(`-j MARK --set-xmark 0x80000/0x0`)},
553+
// do not nat node port service traffic with external traffic policy set to Local
554+
{Table: NAT, Chain: OvnNodePort, Rule: strings.Fields(`-m set --match-set ovn40subnets-distributed-gw dst -j ACCEPT`)},
555+
// nat node port service traffic with external traffic policy set to Local for subnets with centralized gateway
556+
{Table: NAT, Chain: OvnNodePort, Rule: strings.Fields(`-j MASQUERADE`)},
534557
// Input Accept
535558
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)},
536559
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)},
@@ -547,14 +570,12 @@ func (c *Controller) setIptables() error {
547570
v6Rules = []util.IPTableRule{
548571
// mark packets from pod to service
549572
{Table: NAT, Chain: OvnPrerouting, Rule: strings.Fields(`-i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x4000/0x4000`)},
550-
// nat packets marked by kube-proxy or kube-ovn
551-
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)},
573+
// packets marked by kube-proxy or kube-ovn will be SNATed later by kube-proxy
574+
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j RETURN`)},
575+
// handle node port with external traffic policy set to Local
576+
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j ` + OvnNodePort)},
552577
// nat service traffic
553578
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets src -m set --match-set ovn60subnets dst -j MASQUERADE`)},
554-
// do not nat node port service traffic with external traffic policy set to local
555-
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn60subnets-distributed-gw dst -j RETURN`)},
556-
// nat node port service traffic with external traffic policy set to local for subnets with centralized gateway
557-
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j MASQUERADE`)},
558579
// do not nat reply packets in direct routing
559580
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},
560581
// do not nat route traffic
@@ -565,6 +586,12 @@ func (c *Controller) setIptables() error {
565586
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j MASQUERADE`, OnOutGoingNatMark))},
566587
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j RETURN`, OnOutGoingForwardMark))},
567588
{Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`)},
589+
// clear mark
590+
{Table: NAT, Chain: OvnNodePort, Rule: strings.Fields(`-j MARK --set-xmark 0x80000/0x0`)},
591+
// do not nat node port service traffic with external traffic policy set to Local
592+
{Table: NAT, Chain: OvnNodePort, Rule: strings.Fields(`-m set --match-set ovn60subnets-distributed-gw dst -j ACCEPT`)},
593+
// nat node port service traffic with external traffic policy set to Local for subnets with centralized gateway
594+
{Table: NAT, Chain: OvnNodePort, Rule: strings.Fields(`-j MASQUERADE`)},
568595
// Input Accept
569596
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)},
570597
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)},
@@ -682,7 +709,7 @@ func (c *Controller) setIptables() error {
682709
}
683710
}
684711

685-
var natPreroutingRules, natPostroutingRules []util.IPTableRule
712+
var natPreroutingRules, natPostroutingRules, ovnNodePortRules []util.IPTableRule
686713
for _, rule := range iptablesRules {
687714
if rule.Table == NAT {
688715
switch rule.Chain {
@@ -699,6 +726,12 @@ func (c *Controller) setIptables() error {
699726
}
700727
natPostroutingRules = append(natPostroutingRules, rule)
701728
continue
729+
case OvnNodePort:
730+
if c.k8siptables[protocol].HasRandomFully() && rule.Rule[len(rule.Rule)-1] == "MASQUERADE" {
731+
rule.Rule = append(rule.Rule, "--random-fully")
732+
}
733+
ovnNodePortRules = append(ovnNodePortRules, rule)
734+
continue
702735
}
703736
}
704737

@@ -733,6 +766,10 @@ func (c *Controller) setIptables() error {
733766
klog.Errorf("failed to update chain %s/%s: %v", NAT, OvnPrerouting)
734767
return err
735768
}
769+
if err = c.updateIptablesChain(ipt, NAT, OvnNodePort, OvnPostrouting, ovnNodePortRules); err != nil {
770+
klog.Errorf("failed to update chain %s/%s: %v", NAT, OvnNodePort)
771+
return err
772+
}
736773
if err = c.updateIptablesChain(ipt, NAT, OvnPostrouting, Postrouting, natPostroutingRules); err != nil {
737774
klog.Errorf("failed to update chain %s/%s: %v", NAT, OvnPostrouting)
738775
return err

0 commit comments

Comments
 (0)