Skip to content

Commit 87c6868

Browse files
authored
iptables: use the same mode with kube-proxy (#2758)
* iptables: use the same mode with kube-proxy (#2535) * build base images for pr on necessary
1 parent 944f304 commit 87c6868

11 files changed

+532
-114
lines changed

.github/workflows/build-x86-image.yaml

+63-1
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,46 @@ env:
2525
HELM_VERSION: v3.11.1
2626

2727
jobs:
28+
build-kube-ovn-base:
29+
name: Build kube-ovn-base
30+
runs-on: ubuntu-22.04
31+
steps:
32+
- uses: actions/checkout@v3
33+
with:
34+
fetch-depth: 2
35+
- uses: docker/setup-buildx-action@v2
36+
if: github.base_ref != null
37+
- name: Build
38+
run: |
39+
touch .CI_PASSED_VAR
40+
if git diff --name-only HEAD^ HEAD | grep -q ^dist/images/Dockerfile.base$; then
41+
echo "BUILD_BASE=1" > .CI_PASSED_VAR
42+
echo "BUILD_BASE=1" >> "$GITHUB_ENV"
43+
make base-amd64
44+
make base-tar-amd64
45+
fi
46+
if git diff --name-only HEAD^ HEAD | grep -q ^dist/images/Dockerfile.base-dpdk$; then
47+
make base-amd64-dpdk
48+
fi
49+
50+
- name: Upload variable file to artifact
51+
uses: actions/upload-artifact@v3
52+
with:
53+
name: variables
54+
path: .CI_PASSED_VAR
55+
56+
- name: Upload base images to artifact
57+
if: env.BUILD_BASE == 1
58+
uses: actions/upload-artifact@v3
59+
with:
60+
name: kube-ovn-base
61+
path: image-amd64.tar
62+
2863
build-kube-ovn:
2964
name: Build kube-ovn
3065
runs-on: ubuntu-22.04
66+
needs:
67+
- build-kube-ovn-base
3168
steps:
3269
- uses: actions/checkout@v3
3370
- uses: docker/setup-buildx-action@v2
@@ -62,12 +99,37 @@ jobs:
6299
install "$tmp/gosec" /usr/local/bin
63100
rm -rf $tmp
64101
102+
- name: Download variable file
103+
uses: actions/download-artifact@v3
104+
with:
105+
name: variables
106+
107+
- name: Export passed variables
108+
run: cat .CI_PASSED_VAR >> "$GITHUB_ENV"
109+
110+
- name: Download base images
111+
if: env.BUILD_BASE == 1
112+
uses: actions/download-artifact@v3
113+
with:
114+
name: kube-ovn-base
115+
116+
- name: Load base images
117+
if: env.BUILD_BASE == 1
118+
run: docker load --input image-amd64.tar
119+
65120
- name: Build
66121
run: |
67122
go mod tidy
68123
git diff --exit-code
69124
make lint
70-
make image-kube-ovn
125+
if [ "x${{ env.BUILD_BASE }}" = "x1" ]; then
126+
TAG=$(cat VERSION)
127+
docker tag kubeovn/kube-ovn-base:$TAG-amd64 kubeovn/kube-ovn-base:$TAG
128+
docker tag kubeovn/kube-ovn-base:$TAG-amd64-no-avx512 kubeovn/kube-ovn-base:$TAG-no-avx512
129+
make build-kube-ovn
130+
else
131+
make image-kube-ovn
132+
fi
71133
make tar-kube-ovn
72134
73135
- name: Upload images to artifact

Makefile

+16-10
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,15 @@ build-go-arm:
4646
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-cmd -ldflags $(GOLDFLAGS) -v ./cmd
4747
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-webhook -ldflags $(GOLDFLAGS) -v ./cmd/webhook
4848

49+
.PHONY: build-kube-ovn
50+
build-kube-ovn: build-go
51+
docker build -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -f dist/images/Dockerfile dist/images/
52+
docker build -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-no-avx512 -f dist/images/Dockerfile.no-avx512 dist/images/
53+
docker build -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-dpdk -f dist/images/Dockerfile.dpdk dist/images/
54+
4955
.PHONY: build-dev
5056
build-dev: build-go
51-
docker build --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(DEV_TAG) -f dist/images/Dockerfile dist/images/
57+
docker build -t $(REGISTRY)/kube-ovn:$(DEV_TAG) -f dist/images/Dockerfile dist/images/
5258

5359
.PHONY: build-dpdk
5460
build-dpdk:
@@ -69,34 +75,34 @@ base-arm64:
6975

7076
.PHONY: image-kube-ovn
7177
image-kube-ovn: build-go
72-
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
73-
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-no-avx512 -o type=docker -f dist/images/Dockerfile.no-avx512 dist/images/
74-
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-dpdk -o type=docker -f dist/images/Dockerfile.dpdk dist/images/
78+
docker buildx build --platform linux/amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
79+
docker buildx build --platform linux/amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-no-avx512 -o type=docker -f dist/images/Dockerfile.no-avx512 dist/images/
80+
docker buildx build --platform linux/amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-dpdk -o type=docker -f dist/images/Dockerfile.dpdk dist/images/
7581

7682
.PHONY: image-debug
7783
image-debug: build-go
7884
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:debug -o type=docker -f dist/images/Dockerfile.debug dist/images/
7985

8086
.PHONY: image-vpc-nat-gateway
8187
image-vpc-nat-gateway:
82-
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
88+
docker buildx build --platform linux/amd64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
8389

8490
.PHONY: image-centos-compile
8591
image-centos-compile:
86-
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/centos7-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos7/Dockerfile fastpath/
87-
# docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/centos8-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos8/Dockerfile fastpath/
92+
docker buildx build --platform linux/amd64 -t $(REGISTRY)/centos7-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos7/Dockerfile fastpath/
93+
# docker buildx build --platform linux/amd64 -t $(REGISTRY)/centos8-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos8/Dockerfile fastpath/
8894

8995
.PHOONY: image-test
9096
image-test: build-go
91-
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/test:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile.test dist/images/
97+
docker buildx build --platform linux/amd64 -t $(REGISTRY)/test:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile.test dist/images/
9298

9399
.PHONY: release
94100
release: lint image-kube-ovn image-vpc-nat-gateway image-centos-compile
95101

96102
.PHONY: release-arm
97103
release-arm: build-go-arm
98-
docker buildx build --platform linux/arm64 --build-arg ARCH=arm64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
99-
docker buildx build --platform linux/arm64 --build-arg ARCH=arm64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
104+
docker buildx build --platform linux/arm64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
105+
docker buildx build --platform linux/arm64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
100106

101107
.PHONY: push-dev
102108
push-dev:

dist/images/Dockerfile

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ COPY grace_stop_ovn_controller /usr/share/ovn/scripts/grace_stop_ovn_controller
99

1010
WORKDIR /kube-ovn
1111

12+
RUN /kube-ovn/iptables-wrapper-installer.sh --no-sanity-check
1213
RUN rm -f /usr/bin/nc &&\
1314
rm -f /usr/bin/netcat
1415
RUN deluser sync

dist/images/Dockerfile.base

-3
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,6 @@ RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname
8383
logrotate dnsutils net-tools strongswan strongswan-pki libcharon-extra-plugins \
8484
libcharon-extauth-plugins libstrongswan-extra-plugins libstrongswan-standard-plugins -y --no-install-recommends && \
8585
rm -rf /var/lib/apt/lists/* && \
86-
cd /usr/sbin && \
87-
ln -sf /usr/sbin/iptables-legacy iptables && \
88-
ln -sf /usr/sbin/ip6tables-legacy ip6tables && \
8986
rm -rf /etc/localtime
9087

9188
RUN mkdir -p /var/run/openvswitch && \
+211
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
#!/bin/sh
2+
3+
# Copyright 2020 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# Usage:
18+
#
19+
# iptables-wrapper-installer.sh [--no-sanity-check]
20+
#
21+
# Installs a wrapper iptables script in a container that will figure out
22+
# whether iptables-legacy or iptables-nft is in use on the host and then
23+
# replaces itself with the correct underlying iptables version.
24+
#
25+
# Unless "--no-sanity-check" is passed, it will first verify that the
26+
# container already contains a suitable version of iptables.
27+
28+
# NOTE: This can only use POSIX /bin/sh features; the build container
29+
# might not contain bash.
30+
31+
# original source:
32+
# https://github.com/kubernetes-sigs/iptables-wrappers/blob/master/iptables-wrapper-installer.sh
33+
34+
set -eu
35+
36+
# Find iptables binary location
37+
if [ -d /usr/sbin -a -e /usr/sbin/iptables ]; then
38+
sbin="/usr/sbin"
39+
elif [ -d /sbin -a -e /sbin/iptables ]; then
40+
sbin="/sbin"
41+
else
42+
echo "ERROR: iptables is not present in either /usr/sbin or /sbin" 1>&2
43+
exit 1
44+
fi
45+
46+
# Determine how the system selects between iptables-legacy and iptables-nft
47+
if [ -x /usr/sbin/alternatives ]; then
48+
# Fedora/SUSE style alternatives
49+
altstyle="fedora"
50+
elif [ -x /usr/sbin/update-alternatives ]; then
51+
# Debian style alternatives
52+
altstyle="debian"
53+
else
54+
# No alternatives system
55+
altstyle="none"
56+
fi
57+
58+
if [ "${1:-}" != "--no-sanity-check" ]; then
59+
# Ensure dependencies are installed
60+
if ! version=$("${sbin}/iptables-nft" --version 2> /dev/null); then
61+
echo "ERROR: iptables-nft is not installed" 1>&2
62+
exit 1
63+
fi
64+
if ! "${sbin}/iptables-legacy" --version > /dev/null 2>&1; then
65+
echo "ERROR: iptables-legacy is not installed" 1>&2
66+
exit 1
67+
fi
68+
69+
case "${version}" in
70+
*v1.8.[0123]\ *)
71+
echo "ERROR: iptables 1.8.0 - 1.8.3 have compatibility bugs." 1>&2
72+
echo " Upgrade to 1.8.4 or newer." 1>&2
73+
exit 1
74+
;;
75+
*)
76+
# 1.8.4+ are OK
77+
;;
78+
esac
79+
fi
80+
81+
# Start creating the wrapper...
82+
rm -f "${sbin}/iptables-wrapper"
83+
cat > "${sbin}/iptables-wrapper" <<EOF
84+
#!/bin/sh
85+
86+
# Copyright 2020 The Kubernetes Authors.
87+
#
88+
# Licensed under the Apache License, Version 2.0 (the "License");
89+
# you may not use this file except in compliance with the License.
90+
# You may obtain a copy of the License at
91+
#
92+
# http://www.apache.org/licenses/LICENSE-2.0
93+
#
94+
# Unless required by applicable law or agreed to in writing, software
95+
# distributed under the License is distributed on an "AS IS" BASIS,
96+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
97+
# See the License for the specific language governing permissions and
98+
# limitations under the License.
99+
100+
# NOTE: This can only use POSIX /bin/sh features; the container image
101+
# might not contain bash.
102+
103+
set -eu
104+
105+
# In kubernetes 1.17 and later, kubelet will have created at least
106+
# one chain in the "mangle" table (either "KUBE-IPTABLES-HINT" or
107+
# "KUBE-KUBELET-CANARY"), so check that first, against
108+
# iptables-nft, because we can check that more efficiently and
109+
# it's more common these days.
110+
nft_kubelet_rules=\$( (iptables-nft-save -t mangle || true; ip6tables-nft-save -t mangle || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
111+
if [ "\${nft_kubelet_rules}" -ne 0 ]; then
112+
mode=nft
113+
else
114+
# Check for kubernetes 1.17-or-later with iptables-legacy. We
115+
# can't pass "-t mangle" to iptables-legacy-save because it would
116+
# cause the kernel to create that table if it didn't already
117+
# exist, which we don't want. So we have to grab all the rules
118+
legacy_kubelet_rules=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
119+
if [ "\${legacy_kubelet_rules}" -ne 0 ]; then
120+
mode=legacy
121+
else
122+
# With older kubernetes releases there may not be any _specific_
123+
# rules we can look for, but we assume that some non-containerized process
124+
# (possibly kubelet) will have created _some_ iptables rules.
125+
num_legacy_lines=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l)
126+
num_nft_lines=\$( (iptables-nft-save || true; ip6tables-nft-save || true) 2>/dev/null | grep '^-' | wc -l)
127+
if [ "\${num_legacy_lines}" -gt "\${num_nft_lines}" ]; then
128+
mode=legacy
129+
else
130+
mode=nft
131+
fi
132+
fi
133+
fi
134+
135+
EOF
136+
137+
# Write out the appropriate alternatives-selection commands
138+
case "${altstyle}" in
139+
fedora)
140+
cat >> "${sbin}/iptables-wrapper" <<EOF
141+
# Update links to point to the selected binaries
142+
alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
143+
EOF
144+
;;
145+
146+
debian)
147+
cat >> "${sbin}/iptables-wrapper" <<EOF
148+
# Update links to point to the selected binaries
149+
update-alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
150+
update-alternatives --set ip6tables "/usr/sbin/ip6tables-\${mode}" > /dev/null || failed=1
151+
EOF
152+
;;
153+
154+
*)
155+
cat >> "${sbin}/iptables-wrapper" <<EOF
156+
# Update links to point to the selected binaries
157+
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
158+
rm -f "${sbin}/\${cmd}"
159+
ln -s "${sbin}/xtables-\${mode}-multi" "${sbin}/\${cmd}"
160+
done 2>/dev/null || failed=1
161+
EOF
162+
;;
163+
esac
164+
165+
# Write out the post-alternatives-selection error checking and final wrap-up
166+
cat >> "${sbin}/iptables-wrapper" <<EOF
167+
if [ "\${failed:-0}" = 1 ]; then
168+
echo "Unable to redirect iptables binaries. (Are you running in an unprivileged pod?)" 1>&2
169+
# fake it, though this will probably also fail if they aren't root
170+
exec "${sbin}/xtables-\${mode}-multi" "\$0" "\$@"
171+
fi
172+
173+
# Now re-exec the original command with the newly-selected alternative
174+
exec "\$0" "\$@"
175+
EOF
176+
chmod +x "${sbin}/iptables-wrapper"
177+
178+
# Now back in the installer script, point the iptables binaries at our
179+
# wrapper
180+
case "${altstyle}" in
181+
fedora)
182+
alternatives \
183+
--install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
184+
--slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
185+
--slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper \
186+
--slave /usr/sbin/ip6tables iptables /usr/sbin/iptables-wrapper \
187+
--slave /usr/sbin/ip6tables-restore iptables-restore /usr/sbin/iptables-wrapper \
188+
--slave /usr/sbin/ip6tables-save iptables-save /usr/sbin/iptables-wrapper
189+
;;
190+
191+
debian)
192+
update-alternatives \
193+
--install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
194+
--slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
195+
--slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper
196+
update-alternatives \
197+
--install /usr/sbin/ip6tables ip6tables /usr/sbin/iptables-wrapper 100 \
198+
--slave /usr/sbin/ip6tables-restore ip6tables-restore /usr/sbin/iptables-wrapper \
199+
--slave /usr/sbin/ip6tables-save ip6tables-save /usr/sbin/iptables-wrapper
200+
;;
201+
202+
*)
203+
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
204+
rm -f "${sbin}/${cmd}"
205+
ln -s "${sbin}/iptables-wrapper" "${sbin}/${cmd}"
206+
done
207+
;;
208+
esac
209+
210+
# Cleanup
211+
rm -f "$0"

dist/images/start-cniserver.sh

+3
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ do
4141
fi
4242
done
4343

44+
# update links to point to the iptables binaries
45+
iptables -V
46+
4447
# If nftables not exist do not exit
4548
set +e
4649
iptables -P FORWARD ACCEPT

dist/images/start-ovs.sh

+3
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ function quit {
5050
}
5151
trap quit EXIT
5252

53+
# update links to point to the iptables binaries
54+
iptables -V
55+
5356
# Start ovsdb
5457
/usr/share/openvswitch/scripts/ovs-ctl restart --no-ovs-vswitchd --system-id=random
5558
# Restrict the number of pthreads ovs-vswitchd creates to reduce the

0 commit comments

Comments
 (0)