Skip to content

Commit d8e59ab

Browse files
zhangzujianzhangzujian
authored
e2e: do not import pkg/daemon (#3055)
* e2e: do not import `pkg/daemon` * e2e: replace nginx with agnhost * ci: fix kube-ovn installation in schedule e2e
1 parent 20a6526 commit d8e59ab

File tree

6 files changed

+97
-85
lines changed

6 files changed

+97
-85
lines changed

‎.github/workflows/scheduled-e2e.yaml

+2-5
Original file line numberDiff line numberDiff line change
@@ -1360,16 +1360,13 @@ jobs:
13601360
sudo cp -r /root/.kube/ ~/.kube/
13611361
sudo chown -R $(id -un). ~/.kube/
13621362
1363-
- name: Install Kube-OVN
1363+
- name: Install Kube-OVN with VPC NAT gateway enabled
13641364
working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
13651365
run: |
13661366
version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"')
13671367
docker pull kubeovn/kube-ovn:$version
13681368
docker pull kubeovn/vpc-nat-gateway:$version
1369-
VERSION=$version make kind-install
1370-
1371-
- name: Install vpc-nat-gw
1372-
run: make kind-install-vpc-nat-gw
1369+
VERSION=$version make kind-install-vpc-nat-gw
13731370
13741371
- name: Run E2E
13751372
run: make iptables-vpc-nat-gw-conformance-e2e

‎pkg/daemon/gateway_linux.go

+15-15
Original file line numberDiff line numberDiff line change
@@ -39,26 +39,26 @@ const (
3939
)
4040

4141
const (
42-
NAT = "nat"
43-
MANGLE = "mangle"
44-
Prerouting = "PREROUTING"
45-
Postrouting = "POSTROUTING"
46-
Output = "OUTPUT"
47-
OvnPrerouting = "OVN-PREROUTING"
48-
OvnPostrouting = "OVN-POSTROUTING"
49-
OvnOutput = "OVN-OUTPUT"
50-
OvnMasquerade = "OVN-MASQUERADE"
51-
OvnNatOutGoingPolicy = "OVN-NAT-POLICY"
52-
OvnNatOutGoingPolicySubnet = "OVN-NAT-PSUBNET-"
42+
NAT = util.NAT
43+
MANGLE = util.Mangle
44+
Prerouting = util.Prerouting
45+
Postrouting = util.Postrouting
46+
Output = util.Output
47+
OvnPrerouting = util.OvnPrerouting
48+
OvnPostrouting = util.OvnPostrouting
49+
OvnOutput = util.OvnOutput
50+
OvnMasquerade = util.OvnMasquerade
51+
OvnNatOutGoingPolicy = util.OvnNatOutGoingPolicy
52+
OvnNatOutGoingPolicySubnet = util.OvnNatOutGoingPolicySubnet
5353
)
5454

5555
const (
5656
OnOutGoingNatMark = "0x90001/0x90001"
5757
OnOutGoingForwardMark = "0x90002/0x90002"
58-
TProxyOutputMark = 0x90003
59-
TProxyOutputMask = 0x90003
60-
TProxyPreroutingMark = 0x90004
61-
TProxyPreroutingMask = 0x90004
58+
TProxyOutputMark = util.TProxyOutputMark
59+
TProxyOutputMask = util.TProxyOutputMask
60+
TProxyPreroutingMark = util.TProxyPreroutingMark
61+
TProxyPreroutingMask = util.TProxyPreroutingMask
6262
)
6363

6464
type policyRouteMeta struct {

‎pkg/util/const.go

+17
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,23 @@ const (
255255
NatPolicyRuleActionForward = "forward"
256256
NatPolicyRuleIDLength = 12
257257

258+
NAT = "nat"
259+
Mangle = "mangle"
260+
Prerouting = "PREROUTING"
261+
Postrouting = "POSTROUTING"
262+
Output = "OUTPUT"
263+
OvnPrerouting = "OVN-PREROUTING"
264+
OvnPostrouting = "OVN-POSTROUTING"
265+
OvnOutput = "OVN-OUTPUT"
266+
OvnMasquerade = "OVN-MASQUERADE"
267+
OvnNatOutGoingPolicy = "OVN-NAT-POLICY"
268+
OvnNatOutGoingPolicySubnet = "OVN-NAT-PSUBNET-"
269+
258270
TProxyListenPort = 8102
259271
TProxyRouteTable = 10001
272+
273+
TProxyOutputMark = 0x90003
274+
TProxyOutputMask = 0x90003
275+
TProxyPreroutingMark = 0x90004
276+
TProxyPreroutingMask = 0x90004
260277
)

‎test/e2e/framework/image.go

-1
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,4 @@ const (
44
PauseImage = "kubeovn/pause:3.2"
55
BusyBoxImage = "busybox:stable"
66
AgnhostImage = "kubeovn/agnhost:2.43"
7-
NginxImage = "nginx:latest"
87
)

‎test/e2e/kube-ovn/ipam/ipam.go

+1
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ var _ = framework.Describe("[group:ipam]", func() {
253253
}
254254

255255
for replicas := 1; replicas <= 3; replicas++ {
256+
stsName = "sts-" + framework.RandomSuffix()
256257
ippool := framework.RandomIPs(cidr, ippoolSep, replicas)
257258
labels := map[string]string{"app": stsName}
258259

‎test/e2e/kube-ovn/pod/vpc_pod_probe.go

+62-64
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,17 @@ package pod
22

33
import (
44
"fmt"
5-
"time"
5+
"math/rand"
6+
"net"
7+
"strconv"
8+
"strings"
69

710
"github.com/onsi/ginkgo/v2"
811
corev1 "k8s.io/api/core/v1"
912
"k8s.io/apimachinery/pkg/util/intstr"
1013
clientset "k8s.io/client-go/kubernetes"
1114

1215
apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
13-
"github.com/kubeovn/kube-ovn/pkg/daemon"
1416
"github.com/kubeovn/kube-ovn/pkg/util"
1517
"github.com/kubeovn/kube-ovn/test/e2e/framework"
1618
"github.com/kubeovn/kube-ovn/test/e2e/framework/iptables"
@@ -21,6 +23,7 @@ var _ = framework.SerialDescribe("[group:pod]", func() {
2123

2224
var cs clientset.Interface
2325
var podClient *framework.PodClient
26+
var eventClient *framework.EventClient
2427
var subnetClient *framework.SubnetClient
2528
var vpcClient *framework.VpcClient
2629
var namespaceName, subnetName, podName, vpcName string
@@ -31,6 +34,7 @@ var _ = framework.SerialDescribe("[group:pod]", func() {
3134
ginkgo.BeforeEach(func() {
3235
cs = f.ClientSet
3336
podClient = f.PodClient()
37+
eventClient = f.EventClient()
3438
subnetClient = f.SubnetClient()
3539
namespaceName = f.Namespace.Name
3640
subnetName = "subnet-" + framework.RandomSuffix()
@@ -64,7 +68,7 @@ var _ = framework.SerialDescribe("[group:pod]", func() {
6468
}
6569
})
6670

67-
framework.ConformanceIt("should support http and tcp liveness probe and readiness probe in custom vpc pod ", func() {
71+
framework.ConformanceIt("should support http and tcp readiness probe in custom vpc pod", func() {
6872
f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12")
6973
daemonSetClient := f.DaemonSetClientNS(framework.KubeOvnNamespace)
7074
originDs := daemonSetClient.Get("kube-ovn-cni")
@@ -94,108 +98,102 @@ var _ = framework.SerialDescribe("[group:pod]", func() {
9498
subnet := framework.MakeSubnet(custVPCSubnetName, "", cidr, "", vpcName, "", nil, nil, nil)
9599
_ = subnetClient.CreateSync(subnet)
96100

97-
ginkgo.By("Creating pod with HTTP liveness and readiness probe that port is accessible " + podName)
98-
pod := framework.MakePod(namespaceName, podName, nil, map[string]string{util.LogicalSwitchAnnotation: custVPCSubnetName}, framework.NginxImage, nil, nil)
99-
101+
ginkgo.By("Creating pod with HTTP readiness probe that port is accessible " + podName)
102+
port := 8000 + rand.Intn(1000)
103+
portStr := strconv.Itoa(port)
104+
args := []string{"netexec", "--http-port", portStr}
105+
pod := framework.MakePod(namespaceName, podName, nil, map[string]string{util.LogicalSwitchAnnotation: custVPCSubnetName}, framework.AgnhostImage, nil, args)
100106
pod.Spec.Containers[0].ReadinessProbe = &corev1.Probe{
101107
ProbeHandler: corev1.ProbeHandler{
102108
HTTPGet: &corev1.HTTPGetAction{
103-
Port: intstr.FromInt(80),
109+
Port: intstr.FromInt(port),
104110
},
105111
},
106112
}
107-
pod.Spec.Containers[0].LivenessProbe = &corev1.Probe{
108-
ProbeHandler: corev1.ProbeHandler{
109-
HTTPGet: &corev1.HTTPGetAction{
110-
Port: intstr.FromInt(80),
111-
},
112-
},
113-
}
114-
115113
pod = podClient.CreateSync(pod)
116-
framework.ExpectEqual(pod.Status.ContainerStatuses[0].Ready, true)
117-
checkTProxyRules(f, pod, 80, true)
114+
checkTProxyRules(f, pod, port, true)
115+
116+
ginkgo.By("Deleting pod " + podName)
118117
podClient.DeleteSync(podName)
119118

120-
ginkgo.By("Creating pod with HTTP liveness and readiness probe that port is not accessible " + podName)
121-
pod = framework.MakePod(namespaceName, podName, nil, map[string]string{util.LogicalSwitchAnnotation: custVPCSubnetName}, framework.NginxImage, nil, nil)
119+
ginkgo.By("Creating pod with HTTP readiness probe that port is not accessible " + podName)
120+
pod = framework.MakePod(namespaceName, podName, nil, map[string]string{util.LogicalSwitchAnnotation: custVPCSubnetName}, framework.AgnhostImage, nil, args)
122121
pod.Spec.Containers[0].ReadinessProbe = &corev1.Probe{
123122
ProbeHandler: corev1.ProbeHandler{
124123
HTTPGet: &corev1.HTTPGetAction{
125-
Port: intstr.FromInt(81),
124+
Port: intstr.FromInt(port + 1),
126125
},
127126
},
128127
}
129-
pod.Spec.Containers[0].LivenessProbe = &corev1.Probe{
130-
ProbeHandler: corev1.ProbeHandler{
131-
HTTPGet: &corev1.HTTPGetAction{
132-
Port: intstr.FromInt(81),
133-
},
134-
},
128+
_ = podClient.Create(pod)
129+
130+
ginkgo.By("Waiting for pod readiness probe failure")
131+
events := eventClient.WaitToHaveEvent("Pod", podName, "Warning", "Unhealthy", "kubelet", "")
132+
var found bool
133+
for _, event := range events {
134+
if strings.Contains(event.Message, "Readiness probe failed") {
135+
found = true
136+
framework.Logf("Found pod event: %s", event.Message)
137+
break
138+
}
135139
}
140+
framework.ExpectTrue(found, "Pod readiness probe is expected to fail")
136141

137-
_ = podClient.Create(pod)
138-
time.Sleep(5 * time.Second)
139142
pod = podClient.GetPod(podName)
143+
checkTProxyRules(f, pod, port+1, true)
140144

141-
framework.ExpectEqual(pod.Status.ContainerStatuses[0].Ready, false)
142-
checkTProxyRules(f, pod, 81, true)
145+
ginkgo.By("Deleting pod " + podName)
143146
podClient.DeleteSync(podName)
144147

145-
ginkgo.By("Creating pod with TCP probe liveness and readiness probe that port is accessible " + podName)
146-
pod = framework.MakePod(namespaceName, podName, nil, map[string]string{util.LogicalSwitchAnnotation: custVPCSubnetName}, framework.NginxImage, nil, nil)
148+
ginkgo.By("Creating pod with TCP readiness probe that port is accessible " + podName)
149+
pod = framework.MakePod(namespaceName, podName, nil, map[string]string{util.LogicalSwitchAnnotation: custVPCSubnetName}, framework.AgnhostImage, nil, args)
147150
pod.Spec.Containers[0].ReadinessProbe = &corev1.Probe{
148151
ProbeHandler: corev1.ProbeHandler{
149152
TCPSocket: &corev1.TCPSocketAction{
150-
Port: intstr.FromInt(80),
151-
},
152-
},
153-
}
154-
pod.Spec.Containers[0].LivenessProbe = &corev1.Probe{
155-
ProbeHandler: corev1.ProbeHandler{
156-
TCPSocket: &corev1.TCPSocketAction{
157-
Port: intstr.FromInt(80),
153+
Port: intstr.FromInt(port),
158154
},
159155
},
160156
}
161-
162157
pod = podClient.CreateSync(pod)
163-
framework.ExpectEqual(pod.Status.ContainerStatuses[0].Ready, true)
158+
checkTProxyRules(f, pod, port, true)
164159

165-
checkTProxyRules(f, pod, 80, true)
160+
ginkgo.By("Deleting pod " + podName)
166161
podClient.DeleteSync(podName)
167162

168-
ginkgo.By("Creating pod with TCP probe liveness and readiness probe that port is not accessible " + podName)
169-
pod = framework.MakePod(namespaceName, podName, nil, map[string]string{util.LogicalSwitchAnnotation: custVPCSubnetName}, framework.NginxImage, nil, nil)
163+
ginkgo.By("Creating pod with TCP readiness probe that port is not accessible " + podName)
164+
pod = framework.MakePod(namespaceName, podName, nil, map[string]string{util.LogicalSwitchAnnotation: custVPCSubnetName}, framework.AgnhostImage, nil, args)
170165
pod.Spec.Containers[0].ReadinessProbe = &corev1.Probe{
171166
ProbeHandler: corev1.ProbeHandler{
172167
TCPSocket: &corev1.TCPSocketAction{
173-
Port: intstr.FromInt(81),
168+
Port: intstr.FromInt(port - 1),
174169
},
175170
},
176171
}
177-
pod.Spec.Containers[0].LivenessProbe = &corev1.Probe{
178-
ProbeHandler: corev1.ProbeHandler{
179-
TCPSocket: &corev1.TCPSocketAction{
180-
Port: intstr.FromInt(81),
181-
},
182-
},
183-
}
184-
185172
_ = podClient.Create(pod)
186-
time.Sleep(5 * time.Second)
173+
podClient.WaitForRunning(podName)
174+
175+
ginkgo.By("Waiting for pod readiness probe failure")
176+
events = eventClient.WaitToHaveEvent("Pod", podName, "Warning", "Unhealthy", "kubelet", "")
177+
found = false
178+
for _, event := range events {
179+
if strings.Contains(event.Message, "Readiness probe failed") {
180+
found = true
181+
framework.Logf("Found pod event: %s", event.Message)
182+
break
183+
}
184+
}
185+
framework.ExpectTrue(found, "Pod readiness probe is expected to fail")
187186

188187
pod = podClient.GetPod(podName)
189-
framework.ExpectEqual(pod.Status.ContainerStatuses[0].Ready, false)
190-
checkTProxyRules(f, pod, 81, false)
188+
checkTProxyRules(f, pod, port-1, false)
191189
})
192190
})
193191

194192
func checkTProxyRules(f *framework.Framework, pod *corev1.Pod, probePort int, exist bool) {
195193

196194
nodeName := pod.Spec.NodeName
197-
tProxyOutputMarkMask := fmt.Sprintf("%#x/%#x", daemon.TProxyOutputMark, daemon.TProxyOutputMask)
198-
tProxyPreRoutingMarkMask := fmt.Sprintf("%#x/%#x", daemon.TProxyPreroutingMark, daemon.TProxyPreroutingMask)
195+
tProxyOutputMarkMask := fmt.Sprintf("%#x/%#x", util.TProxyOutputMark, util.TProxyOutputMask)
196+
tProxyPreRoutingMarkMask := fmt.Sprintf("%#x/%#x", util.TProxyPreroutingMark, util.TProxyPreroutingMask)
199197

200198
isZeroIP := false
201199
if len(pod.Status.PodIPs) == 2 {
@@ -207,20 +205,20 @@ func checkTProxyRules(f *framework.Framework, pod *corev1.Pod, probePort int, ex
207205
expectedRules := []string{
208206
fmt.Sprintf(`-A OVN-OUTPUT -d %s/32 -p tcp -m tcp --dport %d -j MARK --set-xmark %s`, podIP.IP, probePort, tProxyOutputMarkMask),
209207
}
210-
iptables.CheckIptablesRulesOnNode(f, nodeName, daemon.MANGLE, daemon.OvnOutput, apiv1.ProtocolIPv4, expectedRules, exist)
208+
iptables.CheckIptablesRulesOnNode(f, nodeName, util.Mangle, util.OvnOutput, apiv1.ProtocolIPv4, expectedRules, exist)
211209
hostIP := pod.Status.HostIP
212210
if isZeroIP {
213-
hostIP = "0.0.0.0"
211+
hostIP = net.IPv4zero.String()
214212
}
215213
expectedRules = []string{
216214
fmt.Sprintf(`-A OVN-PREROUTING -d %s/32 -p tcp -m tcp --dport %d -j TPROXY --on-port %d --on-ip %s --tproxy-mark %s`, podIP.IP, probePort, util.TProxyListenPort, hostIP, tProxyPreRoutingMarkMask),
217215
}
218-
iptables.CheckIptablesRulesOnNode(f, nodeName, daemon.MANGLE, daemon.OvnPrerouting, apiv1.ProtocolIPv4, expectedRules, exist)
216+
iptables.CheckIptablesRulesOnNode(f, nodeName, util.Mangle, util.OvnPrerouting, apiv1.ProtocolIPv4, expectedRules, exist)
219217
} else if util.CheckProtocol(podIP.IP) == apiv1.ProtocolIPv6 {
220218
expectedRules := []string{
221219
fmt.Sprintf(`-A OVN-OUTPUT -d %s/128 -p tcp -m tcp --dport %d -j MARK --set-xmark %s`, podIP.IP, probePort, tProxyOutputMarkMask),
222220
}
223-
iptables.CheckIptablesRulesOnNode(f, nodeName, daemon.MANGLE, daemon.OvnOutput, apiv1.ProtocolIPv6, expectedRules, exist)
221+
iptables.CheckIptablesRulesOnNode(f, nodeName, util.Mangle, util.OvnOutput, apiv1.ProtocolIPv6, expectedRules, exist)
224222

225223
hostIP := pod.Status.HostIP
226224
if isZeroIP {
@@ -229,7 +227,7 @@ func checkTProxyRules(f *framework.Framework, pod *corev1.Pod, probePort int, ex
229227
expectedRules = []string{
230228
fmt.Sprintf(`-A OVN-PREROUTING -d %s/128 -p tcp -m tcp --dport %d -j TPROXY --on-port %d --on-ip %s --tproxy-mark %s`, podIP.IP, probePort, util.TProxyListenPort, hostIP, tProxyPreRoutingMarkMask),
231229
}
232-
iptables.CheckIptablesRulesOnNode(f, nodeName, daemon.MANGLE, daemon.OvnPrerouting, apiv1.ProtocolIPv6, expectedRules, exist)
230+
iptables.CheckIptablesRulesOnNode(f, nodeName, util.Mangle, util.OvnPrerouting, apiv1.ProtocolIPv6, expectedRules, exist)
233231
}
234232
}
235233
}

0 commit comments

Comments
 (0)