Skip to content

Commit bd4d99b

Browse files
zhangzujianzhangzujian
committed
add e2e test for ovn db recover (#3118)
1 parent 74f69b2 commit bd4d99b

File tree

9 files changed

+264
-115
lines changed

9 files changed

+264
-115
lines changed

‎.github/workflows/build-x86-image.yaml

+102-99
Original file line numberDiff line numberDiff line change
@@ -1654,8 +1654,8 @@ jobs:
16541654
- name: Cleanup
16551655
run: sh -x dist/images/cleanup.sh
16561656

1657-
kube-ovn-security-e2e:
1658-
name: Kube-OVN Security E2E
1657+
kube-ovn-ha-e2e:
1658+
name: Kube-OVN HA E2E
16591659
needs:
16601660
- build-kube-ovn
16611661
- build-e2e-binaries
@@ -1752,117 +1752,26 @@ jobs:
17521752
env:
17531753
E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
17541754
E2E_IP_FAMILY: ${{ matrix.ip-family }}
1755-
run: make kube-ovn-security-e2e
1755+
run: |
1756+
make kube-ovn-security-e2e
1757+
make kube-ovn-ha-e2e
17561758
17571759
- name: kubectl ko log
17581760
if: failure()
17591761
run: |
17601762
make kubectl-ko-log
1761-
mv kubectl-ko-log.tar.gz kube-ovn-security-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log.tar.gz
1763+
mv kubectl-ko-log.tar.gz kube-ovn-ha-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log.tar.gz
17621764
17631765
- name: upload kubectl ko log
17641766
uses: actions/upload-artifact@v3
17651767
if: failure()
17661768
with:
1767-
name: kube-ovn-security-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log
1768-
path: kube-ovn-security-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log.tar.gz
1769+
name: kube-ovn-ha-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log
1770+
path: kube-ovn-ha-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log.tar.gz
17691771

17701772
- name: Cleanup
17711773
run: sh -x dist/images/cleanup.sh
17721774

1773-
push:
1774-
name: Push Images
1775-
needs:
1776-
- build-centos-compile
1777-
- k8s-conformance-e2e
1778-
- k8s-netpol-e2e
1779-
- k8s-netpol-legacy-e2e
1780-
- cyclonus-netpol-e2e
1781-
- kube-ovn-conformance-e2e
1782-
- kube-ovn-ic-conformance-e2e
1783-
- ovn-vpc-nat-gw-conformance-e2e
1784-
- iptables-vpc-nat-gw-conformance-e2e
1785-
- webhook-e2e
1786-
- lb-svc-e2e
1787-
- underlay-logical-gateway-installation-test
1788-
- chart-installation-test
1789-
- installation-compatibility-test
1790-
- no-ovn-lb-test
1791-
- no-np-test
1792-
- cilium-chaining-e2e
1793-
- kube-ovn-security-e2e
1794-
- kubevirt-e2e
1795-
if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
1796-
runs-on: ubuntu-22.04
1797-
steps:
1798-
- uses: actions/checkout@v3
1799-
1800-
- name: Download kube-ovn image
1801-
uses: actions/download-artifact@v3
1802-
with:
1803-
name: kube-ovn
1804-
1805-
- name: Download vpc-nat-gateway image
1806-
uses: actions/download-artifact@v3
1807-
with:
1808-
name: vpc-nat-gateway
1809-
1810-
- name: Download centos7-compile image
1811-
uses: actions/download-artifact@v3
1812-
with:
1813-
name: centos7-compile
1814-
1815-
# - name: Download centos8-compile image
1816-
# uses: actions/download-artifact@v3
1817-
# with:
1818-
# name: centos8-compile
1819-
1820-
- name: Load image
1821-
run: |
1822-
docker load --input kube-ovn.tar
1823-
docker load --input vpc-nat-gateway.tar
1824-
docker load --input centos7-compile.tar
1825-
# docker load --input centos8-compile.tar
1826-
1827-
- name: Security Scan
1828-
run: |
1829-
sudo apt-get install wget apt-transport-https gnupg lsb-release
1830-
wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
1831-
echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list
1832-
sudo apt-get update
1833-
sudo apt-get install trivy
1834-
make scan
1835-
1836-
- name: Push
1837-
if: github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, 'release-')
1838-
env:
1839-
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
1840-
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
1841-
COMMIT: ${{ github.sha }}
1842-
run: |
1843-
cat VERSION
1844-
TAG=$(cat VERSION)
1845-
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
1846-
docker tag kubeovn/kube-ovn:$TAG kubeovn/kube-ovn-dev:$COMMIT-x86
1847-
docker tag kubeovn/kube-ovn:$TAG kubeovn/kube-ovn:$TAG-x86
1848-
docker tag kubeovn/kube-ovn:$TAG-debug kubeovn/kube-ovn:$TAG-debug-x86
1849-
docker tag kubeovn/vpc-nat-gateway:$TAG kubeovn/vpc-nat-gateway-dev:$COMMIT-x86
1850-
docker tag kubeovn/vpc-nat-gateway:$TAG kubeovn/vpc-nat-gateway:$TAG-x86
1851-
docker tag kubeovn/centos7-compile:$TAG kubeovn/centos7-compile-dev:$TAG-x86
1852-
docker tag kubeovn/centos7-compile:$TAG kubeovn/centos7-compile:$TAG-x86
1853-
# docker tag kubeovn/centos8-compile:$TAG kubeovn/centos8-compile-dev:$TAG-x86
1854-
# docker tag kubeovn/centos8-compile:$TAG kubeovn/centos8-compile:$TAG-x86
1855-
docker images
1856-
docker push kubeovn/kube-ovn:$TAG-x86
1857-
docker push kubeovn/kube-ovn-dev:$COMMIT-x86
1858-
docker push kubeovn/kube-ovn:$TAG-debug-x86
1859-
docker push kubeovn/vpc-nat-gateway:$TAG-x86
1860-
docker push kubeovn/vpc-nat-gateway-dev:$COMMIT-x86
1861-
docker push kubeovn/centos7-compile:$TAG-x86
1862-
docker push kubeovn/centos7-compile-dev:$TAG-x86
1863-
# docker push kubeovn/centos8-compile:$TAG-x86
1864-
# docker push kubeovn/centos8-compile-dev:$TAG-x86
1865-
18661775
kube-ovn-submariner-conformance-e2e:
18671776
name: Kube-OVN Submariner Conformance E2E
18681777
needs:
@@ -2114,3 +2023,97 @@ jobs:
21142023
- name: Run E2E
21152024
working-directory: ${{ env.E2E_DIR }}
21162025
run: make ovn-vpc-nat-gw-conformance-e2e
2026+
2027+
push:
2028+
name: Push Images
2029+
needs:
2030+
- build-centos-compile
2031+
- k8s-conformance-e2e
2032+
- k8s-netpol-e2e
2033+
- k8s-netpol-legacy-e2e
2034+
- cyclonus-netpol-e2e
2035+
- kube-ovn-conformance-e2e
2036+
- kube-ovn-ic-conformance-e2e
2037+
- ovn-vpc-nat-gw-conformance-e2e
2038+
- iptables-vpc-nat-gw-conformance-e2e
2039+
- webhook-e2e
2040+
- lb-svc-e2e
2041+
- underlay-logical-gateway-installation-test
2042+
- chart-installation-test
2043+
- installation-compatibility-test
2044+
- no-ovn-lb-test
2045+
- no-np-test
2046+
- cilium-chaining-e2e
2047+
- kube-ovn-ha-e2e
2048+
- kubevirt-e2e
2049+
- kube-ovn-submariner-conformance-e2e
2050+
if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
2051+
runs-on: ubuntu-22.04
2052+
steps:
2053+
- uses: actions/checkout@v3
2054+
2055+
- name: Download kube-ovn image
2056+
uses: actions/download-artifact@v3
2057+
with:
2058+
name: kube-ovn
2059+
2060+
- name: Download vpc-nat-gateway image
2061+
uses: actions/download-artifact@v3
2062+
with:
2063+
name: vpc-nat-gateway
2064+
2065+
- name: Download centos7-compile image
2066+
uses: actions/download-artifact@v3
2067+
with:
2068+
name: centos7-compile
2069+
2070+
# - name: Download centos8-compile image
2071+
# uses: actions/download-artifact@v3
2072+
# with:
2073+
# name: centos8-compile
2074+
2075+
- name: Load image
2076+
run: |
2077+
docker load --input kube-ovn.tar
2078+
docker load --input vpc-nat-gateway.tar
2079+
docker load --input centos7-compile.tar
2080+
# docker load --input centos8-compile.tar
2081+
2082+
- name: Security Scan
2083+
run: |
2084+
sudo apt-get install wget apt-transport-https gnupg lsb-release
2085+
wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
2086+
echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list
2087+
sudo apt-get update
2088+
sudo apt-get install trivy
2089+
make scan
2090+
2091+
- name: Push
2092+
if: github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, 'release-')
2093+
env:
2094+
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
2095+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
2096+
COMMIT: ${{ github.sha }}
2097+
run: |
2098+
cat VERSION
2099+
TAG=$(cat VERSION)
2100+
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
2101+
docker tag kubeovn/kube-ovn:$TAG kubeovn/kube-ovn-dev:$COMMIT-x86
2102+
docker tag kubeovn/kube-ovn:$TAG kubeovn/kube-ovn:$TAG-x86
2103+
docker tag kubeovn/kube-ovn:$TAG-debug kubeovn/kube-ovn:$TAG-debug-x86
2104+
docker tag kubeovn/vpc-nat-gateway:$TAG kubeovn/vpc-nat-gateway-dev:$COMMIT-x86
2105+
docker tag kubeovn/vpc-nat-gateway:$TAG kubeovn/vpc-nat-gateway:$TAG-x86
2106+
docker tag kubeovn/centos7-compile:$TAG kubeovn/centos7-compile-dev:$TAG-x86
2107+
docker tag kubeovn/centos7-compile:$TAG kubeovn/centos7-compile:$TAG-x86
2108+
# docker tag kubeovn/centos8-compile:$TAG kubeovn/centos8-compile-dev:$TAG-x86
2109+
# docker tag kubeovn/centos8-compile:$TAG kubeovn/centos8-compile:$TAG-x86
2110+
docker images
2111+
docker push kubeovn/kube-ovn:$TAG-x86
2112+
docker push kubeovn/kube-ovn-dev:$COMMIT-x86
2113+
docker push kubeovn/kube-ovn:$TAG-debug-x86
2114+
docker push kubeovn/vpc-nat-gateway:$TAG-x86
2115+
docker push kubeovn/vpc-nat-gateway-dev:$COMMIT-x86
2116+
docker push kubeovn/centos7-compile:$TAG-x86
2117+
docker push kubeovn/centos7-compile-dev:$TAG-x86
2118+
# docker push kubeovn/centos8-compile:$TAG-x86
2119+
# docker push kubeovn/centos8-compile-dev:$TAG-x86

‎.github/workflows/scheduled-e2e.yaml

+5-3
Original file line numberDiff line numberDiff line change
@@ -1047,8 +1047,8 @@ jobs:
10471047
working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
10481048
run: sh dist/images/cleanup.sh
10491049

1050-
kube-ovn-security-e2e:
1051-
name: Kube-OVN Security E2E
1050+
kube-ovn-ha-e2e:
1051+
name: Kube-OVN HA E2E
10521052
runs-on: ubuntu-22.04
10531053
timeout-minutes: 30
10541054
strategy:
@@ -1132,7 +1132,9 @@ jobs:
11321132
env:
11331133
E2E_BRANCH: ${{ matrix.branch }}
11341134
E2E_IP_FAMILY: ${{ matrix.ip-family }}
1135-
run: make kube-ovn-security-e2e
1135+
run: |
1136+
make kube-ovn-security-e2e
1137+
make kube-ovn-ha-e2e
11361138
11371139
- name: Cleanup
11381140
run: sh dist/images/cleanup.sh

‎Makefile.e2e

+10
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ e2e-build:
6363
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc
6464
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw
6565
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-vpc-nat-gw
66+
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ha
6667
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/security
6768
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/kubevirt
6869
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/webhook
@@ -155,6 +156,15 @@ ovn-vpc-nat-gw-conformance-e2e:
155156
ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all -v \
156157
--focus=CNI:Kube-OVN ./test/e2e/ovn-vpc-nat-gw/ovn-vpc-nat-gw.test
157158

159+
.PHONY: kube-ovn-ha-e2e
160+
kube-ovn-ha-e2e:
161+
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ha
162+
E2E_BRANCH=$(E2E_BRANCH) \
163+
E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
164+
E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
165+
ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all -v \
166+
--focus=CNI:Kube-OVN ./test/e2e/ha/ha.test
167+
158168
.PHONY: kube-ovn-security-e2e
159169
kube-ovn-security-e2e:
160170
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/security

‎test/e2e/framework/deployment.go

+22
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,15 @@ func (c *DeploymentClient) GetPods(deploy *appsv1.Deployment) (*corev1.PodList,
5454
return deployment.GetPodsForDeployment(context.Background(), c.f.ClientSet, deploy)
5555
}
5656

57+
func (c *DeploymentClient) GetAllPods(deploy *appsv1.Deployment) (*corev1.PodList, error) {
58+
podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector)
59+
if err != nil {
60+
return nil, err
61+
}
62+
podListOptions := metav1.ListOptions{LabelSelector: podSelector.String()}
63+
return c.f.ClientSet.CoreV1().Pods(deploy.Namespace).List(context.TODO(), podListOptions)
64+
}
65+
5766
// Create creates a new deployment according to the framework specifications
5867
func (c *DeploymentClient) Create(deploy *appsv1.Deployment) *appsv1.Deployment {
5968
d, err := c.DeploymentInterface.Create(context.TODO(), deploy, metav1.CreateOptions{})
@@ -151,6 +160,19 @@ func (c *DeploymentClient) RestartSync(deploy *appsv1.Deployment) *appsv1.Deploy
151160
return c.RolloutStatus(deploy.Name)
152161
}
153162

163+
func (c *DeploymentClient) SetScale(deployment string, replicas int32) {
164+
scale, err := c.GetScale(context.Background(), deployment, metav1.GetOptions{})
165+
framework.ExpectNoError(err)
166+
if scale.Spec.Replicas == replicas {
167+
Logf("repliacs of deployment %s/%s has already been set to %d", c.namespace, deployment, replicas)
168+
return
169+
}
170+
171+
scale.Spec.Replicas = replicas
172+
_, err = c.UpdateScale(context.Background(), deployment, scale, metav1.UpdateOptions{})
173+
framework.ExpectNoError(err)
174+
}
175+
154176
// Delete deletes a deployment if the deployment exists
155177
func (c *DeploymentClient) Delete(name string) {
156178
err := c.DeploymentInterface.Delete(context.TODO(), name, metav1.DeleteOptions{})

‎test/e2e/framework/exec_utils.go

+11-11
Original file line numberDiff line numberDiff line change
@@ -12,23 +12,23 @@ import (
1212
)
1313

1414
// ExecCommandInContainer executes a command in the specified container.
15-
func ExecCommandInContainer(f *Framework, podName, containerName string, cmd ...string) (string, string, error) {
16-
return util.ExecuteCommandInContainer(f.ClientSet, f.ClientConfig(), f.Namespace.Name, podName, containerName, cmd...)
15+
func ExecCommandInContainer(f *Framework, namespace, pod, container string, cmd ...string) (string, string, error) {
16+
return util.ExecuteCommandInContainer(f.ClientSet, f.ClientConfig(), namespace, pod, container, cmd...)
1717
}
1818

1919
// ExecShellInContainer executes the specified command on the pod's container.
20-
func ExecShellInContainer(f *Framework, podName, containerName string, cmd string) (string, string, error) {
21-
return ExecCommandInContainer(f, podName, containerName, "/bin/sh", "-c", cmd)
20+
func ExecShellInContainer(f *Framework, namespace, pod, container, cmd string) (string, string, error) {
21+
return ExecCommandInContainer(f, namespace, pod, container, "/bin/sh", "-c", cmd)
2222
}
2323

24-
func execCommandInPod(ctx context.Context, f *Framework, podName string, cmd ...string) (string, string, error) {
25-
pod, err := f.PodClient().Get(ctx, podName, metav1.GetOptions{})
26-
framework.ExpectNoError(err, "failed to get pod %v", podName)
27-
gomega.Expect(pod.Spec.Containers).NotTo(gomega.BeEmpty())
28-
return ExecCommandInContainer(f, podName, pod.Spec.Containers[0].Name, cmd...)
24+
func execCommandInPod(ctx context.Context, f *Framework, namespace, pod string, cmd ...string) (string, string, error) {
25+
p, err := f.PodClientNS(namespace).Get(ctx, pod, metav1.GetOptions{})
26+
framework.ExpectNoError(err, "failed to get pod %s/%s", namespace, pod)
27+
gomega.Expect(p.Spec.Containers).NotTo(gomega.BeEmpty())
28+
return ExecCommandInContainer(f, namespace, pod, p.Spec.Containers[0].Name, cmd...)
2929
}
3030

3131
// ExecShellInPod executes the specified command on the pod.
32-
func ExecShellInPod(ctx context.Context, f *Framework, podName string, cmd string) (string, string, error) {
33-
return execCommandInPod(ctx, f, podName, "/bin/sh", "-c", cmd)
32+
func ExecShellInPod(ctx context.Context, f *Framework, namespace, pod, cmd string) (string, string, error) {
33+
return execCommandInPod(ctx, f, namespace, pod, "/bin/sh", "-c", cmd)
3434
}

‎test/e2e/framework/framework.go

+4
Original file line numberDiff line numberDiff line change
@@ -186,3 +186,7 @@ func OrderedDescribe(text string, body func()) bool {
186186
func ConformanceIt(text string, body interface{}) bool {
187187
return ginkgo.It(text+" [Conformance]", ginkgo.Offset(1), body)
188188
}
189+
190+
func CorruptiveIt(text string, body interface{}) bool {
191+
return ginkgo.It(text+" [Corruptive]", ginkgo.Offset(1), body)
192+
}

0 commit comments

Comments
 (0)