Skip to content

Commit 2a4b980

Browse files
zhangzujianzhangzujian
committed
ovn db: add support for listening on pod ip (#2235)
1 parent 0d88edd commit 2a4b980

File tree

10 files changed

+198
-84
lines changed

10 files changed

+198
-84
lines changed

‎.github/workflows/build-x86-image.yaml

+105-6
Original file line numberDiff line numberDiff line change
@@ -567,8 +567,8 @@ jobs:
567567
run: sh dist/images/cleanup.sh
568568

569569
underlay-logical-gateway-installation-test:
570-
needs: build-kube-ovn
571570
name: Underlay Logical Gateway Installation Test
571+
needs: build-kube-ovn
572572
runs-on: ubuntu-22.04
573573
timeout-minutes: 30
574574
steps:
@@ -603,8 +603,8 @@ jobs:
603603
run: sh dist/images/cleanup.sh
604604

605605
no-ovn-lb-test:
606-
needs: build-kube-ovn
607606
name: Disable OVN LB Test
607+
needs: build-kube-ovn
608608
runs-on: ubuntu-22.04
609609
timeout-minutes: 30
610610
steps:
@@ -641,8 +641,8 @@ jobs:
641641
run: sh dist/images/cleanup.sh
642642

643643
no-np-test:
644-
needs: build-kube-ovn
645644
name: Disable Network Policy Test
645+
needs: build-kube-ovn
646646
runs-on: ubuntu-22.04
647647
timeout-minutes: 30
648648
steps:
@@ -679,8 +679,8 @@ jobs:
679679
run: sh dist/images/cleanup.sh
680680

681681
installation-compatibility-test:
682-
needs: build-kube-ovn
683682
name: Installation Compatibility Test
683+
needs: build-kube-ovn
684684
runs-on: ubuntu-22.04
685685
timeout-minutes: 10
686686
steps:
@@ -715,8 +715,8 @@ jobs:
715715
run: sh dist/images/cleanup.sh
716716

717717
cilium-chaining-e2e:
718-
needs: build-kube-ovn
719718
name: Cilium Chaining E2E
719+
needs: build-kube-ovn
720720
runs-on: ubuntu-22.04
721721
timeout-minutes: 30
722722
steps:
@@ -802,7 +802,106 @@ jobs:
802802
- name: Cleanup
803803
run: sh dist/images/cleanup.sh
804804

805+
kube-ovn-security-e2e:
806+
name: Kube-OVN Security E2E
807+
needs: build-kube-ovn
808+
runs-on: ubuntu-22.04
809+
timeout-minutes: 30
810+
strategy:
811+
fail-fast: false
812+
matrix:
813+
ssl:
814+
- "true"
815+
- "false"
816+
bind-local:
817+
- "true"
818+
- "false"
819+
ip-family:
820+
- ipv4
821+
- ipv6
822+
- dual
823+
steps:
824+
- uses: actions/checkout@v3
825+
826+
- name: Create the default branch directory
827+
run: mkdir -p test/e2e/source
828+
829+
- name: Check out the default branch
830+
uses: actions/checkout@v3
831+
with:
832+
ref: ${{ github.event.repository.default_branch }}
833+
fetch-depth: 1
834+
path: test/e2e/source
835+
836+
- name: Export E2E directory
837+
run: |
838+
if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
839+
echo "E2E_DIR=." >> "$GITHUB_ENV"
840+
else
841+
echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
842+
fi
843+
844+
- uses: actions/setup-go@v3
845+
with:
846+
go-version: '${{ env.GO_VERSION }}'
847+
check-latest: true
848+
id: go
849+
850+
- name: Export Go full version
851+
run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV"
852+
853+
- name: Go cache
854+
uses: actions/cache@v3
855+
with:
856+
path: |
857+
~/.cache/go-build
858+
~/go/pkg/mod
859+
key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles('${{ env.E2E_DIR }}/**/go.sum') }}
860+
restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-
861+
862+
- name: Build e2e binaries
863+
working-directory: ${{ env.E2E_DIR }}
864+
run: make e2e-compile
865+
866+
- name: Install kind
867+
run: |
868+
curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
869+
chmod +x ./kind
870+
sudo mv kind /usr/local/bin
871+
872+
- name: Download image
873+
uses: actions/download-artifact@v3
874+
with:
875+
name: kube-ovn
876+
877+
- name: Load image
878+
run: docker load --input kube-ovn.tar
879+
880+
- name: Create kind cluster
881+
run: |
882+
sudo pip3 install j2cli
883+
sudo pip3 install "j2cli[yaml]"
884+
sudo PATH=~/.local/bin:$PATH make kind-init-ha-${{ matrix.ip-family }}
885+
sudo cp -r /root/.kube/ ~/.kube/
886+
sudo chown -R $(id -un). ~/.kube/
887+
888+
- name: Install Kube-OVN
889+
run: |
890+
sudo ENABLE_SSL=${{ matrix.ssl }} ENABLE_BIND_LOCAL_IP=${{ matrix.bind-local }} \
891+
make kind-install-${{ matrix.ip-family }}
892+
893+
- name: Run E2E
894+
working-directory: ${{ env.E2E_DIR }}
895+
env:
896+
E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
897+
E2E_IP_FAMILY: ${{ matrix.ip-family }}
898+
run: make kube-ovn-security-e2e
899+
900+
- name: Cleanup
901+
run: sh dist/images/cleanup.sh
902+
805903
push:
904+
name: Push Images
806905
needs:
807906
- build-centos-compile
808907
- k8s-conformance-e2e
@@ -816,7 +915,7 @@ jobs:
816915
- no-ovn-lb-test
817916
- no-np-test
818917
- cilium-chaining-e2e
819-
name: push
918+
- kube-ovn-security-e2e
820919
runs-on: ubuntu-22.04
821920
steps:
822921
- uses: actions/checkout@v3

‎Makefile

+12-1
Original file line numberDiff line numberDiff line change
@@ -233,9 +233,20 @@ kind-init-iptables:
233233
@kube_proxy_mode=iptables $(MAKE) kind-init
234234

235235
.PHONY: kind-init-ha
236-
kind-init-ha:
236+
kind-init-ha: kind-init-ha-ipv4
237+
238+
.PHONY: kind-init-ha-ipv4
239+
kind-init-ha-ipv4:
237240
@ha=true $(MAKE) kind-init
238241

242+
.PHONY: kind-init-ha-ipv6
243+
kind-init-ha-ipv6:
244+
@ip_family=ipv6 $(MAKE) kind-init-ha
245+
246+
.PHONY: kind-init-ha-dual
247+
kind-init-ha-dual:
248+
@ip_family=dual $(MAKE) kind-init-ha
249+
239250
.PHONY: kind-init-single
240251
kind-init-single:
241252
@single=true $(MAKE) kind-init

‎dist/images/Dockerfile.base

+2
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ RUN dpkg -i /usr/src/python3-openvswitch*.deb /usr/src/libopenvswitch*.deb
4040

4141
RUN cd /usr/src/ && git clone -b branch-22.03 --depth=1 https://github.com/ovn-org/ovn.git && \
4242
cd ovn && \
43+
# fix ssl listen address
44+
curl -s https://github.com/kubeovn/ovn/commit/62d4969877712c26fe425698d898b440f91b44bf.patch | git apply && \
4345
# ovn-controller: Add a generic way to check if the daemon started recently.
4446
curl -s https://github.com/kubeovn/ovn/commit/367d7ab6239089ce99dda20818c8833c69ffd77f.patch | git apply && \
4547
# patch.c: Avoid patch interface deletion & recreation during restart.

‎dist/images/install.sh

+12
Original file line numberDiff line numberDiff line change
@@ -1649,6 +1649,12 @@ spec:
16491649
valueFrom:
16501650
fieldRef:
16511651
fieldPath: metadata.namespace
1652+
- name: POD_IPS
1653+
valueFrom:
1654+
fieldRef:
1655+
fieldPath: status.podIPs
1656+
- name: ENABLE_BIND_LOCAL_IP
1657+
value: "$ENABLE_BIND_LOCAL_IP"
16521658
resources:
16531659
requests:
16541660
cpu: 300m
@@ -2137,6 +2143,12 @@ spec:
21372143
valueFrom:
21382144
fieldRef:
21392145
fieldPath: metadata.namespace
2146+
- name: POD_IPS
2147+
valueFrom:
2148+
fieldRef:
2149+
fieldPath: status.podIPs
2150+
- name: ENABLE_BIND_LOCAL_IP
2151+
value: "$ENABLE_BIND_LOCAL_IP"
21402152
resources:
21412153
requests:
21422154
cpu: 300m

‎dist/images/ovn-is-leader.sh

+8-6
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@ ovn-ctl status_northd
88
ovn-ctl status_ovnnb
99
ovn-ctl status_ovnsb
1010

11+
BIND_LOCAL_ADDR=[${POD_IP:-127.0.0.1}]
12+
1113
# For data consistency, only store leader address in endpoint
1214
# Store ovn-nb leader to svc kube-system/ovn-nb
1315
if [[ "$ENABLE_SSL" == "false" ]]; then
14-
nb_leader=$(ovsdb-client query tcp:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
16+
nb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
1517
else
16-
nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
18+
nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
1719
fi
1820

1921
if [[ $nb_leader =~ "true" ]]
@@ -34,9 +36,9 @@ fi
3436

3537
# Store ovn-sb leader to svc kube-system/ovn-sb
3638
if [[ "$ENABLE_SSL" == "false" ]]; then
37-
sb_leader=$(ovsdb-client query tcp:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
39+
sb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
3840
else
39-
sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
41+
sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
4042
fi
4143

4244
if [[ $sb_leader =~ "true" ]]
@@ -51,9 +53,9 @@ then
5153
if [ "$northd_leader" == "" ]; then
5254
# no available northd leader try to release the lock
5355
if [[ "$ENABLE_SSL" == "false" ]]; then
54-
ovsdb-client -v -t 1 steal tcp:127.0.0.1:6642 ovn_northd
56+
ovsdb-client -v -t 1 steal tcp:$BIND_LOCAL_ADDR:6642 ovn_northd
5557
else
56-
ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:127.0.0.1:6642 ovn_northd
58+
ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:$BIND_LOCAL_ADDR:6642 ovn_northd
5759
fi
5860
fi
5961
fi

‎dist/images/start-db.sh

+26-10
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,14 @@ DB_NB_PORT=${DB_NB_PORT:-6641}
2525
DB_SB_ADDR=${DB_SB_ADDR:-::}
2626
DB_SB_PORT=${DB_SB_PORT:-6642}
2727
ENABLE_SSL=${ENABLE_SSL:-false}
28+
ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-false}
29+
BIND_LOCAL_ADDR=[::]
30+
if [[ $ENABLE_BIND_LOCAL_IP == "true" ]]; then
31+
POD_IPS_LIST=(${POD_IPS//,/ })
32+
if [[ ${#POD_IPS_LIST[@]} == 1 ]]; then
33+
BIND_LOCAL_ADDR="[${POD_IP}]"
34+
fi
35+
fi
2836

2937
. /usr/share/openvswitch/scripts/ovs-lib || exit 1
3038

@@ -177,8 +185,10 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
177185
--db-sb-create-insecure-remote=yes \
178186
--db-nb-cluster-local-addr="[${POD_IP}]" \
179187
--db-sb-cluster-local-addr="[${POD_IP}]" \
180-
--db-nb-addr=[::] \
181-
--db-sb-addr=[::] \
188+
--db-nb-addr=$BIND_LOCAL_ADDR \
189+
--db-sb-addr=$BIND_LOCAL_ADDR \
190+
--db-nb-use-remote-in-db=no \
191+
--db-sb-use-remote-in-db=no \
182192
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
183193
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
184194
start_northd
@@ -222,8 +232,10 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
222232
--db-sb-cluster-local-addr="[${POD_IP}]" \
223233
--db-nb-cluster-remote-addr="[${nb_leader_ip}]" \
224234
--db-sb-cluster-remote-addr="[${sb_leader_ip}]" \
225-
--db-nb-addr=[::] \
226-
--db-sb-addr=[::] \
235+
--db-nb-addr=$BIND_LOCAL_ADDR \
236+
--db-sb-addr=$BIND_LOCAL_ADDR \
237+
--db-nb-use-remote-in-db=no \
238+
--db-sb-use-remote-in-db=no \
227239
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
228240
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
229241
start_northd
@@ -277,16 +289,18 @@ else
277289
--ovn-northd-ssl-ca-cert=/var/run/tls/cacert \
278290
--db-nb-cluster-local-addr="[${POD_IP}]" \
279291
--db-sb-cluster-local-addr="[${POD_IP}]" \
280-
--db-nb-addr=[::] \
281-
--db-sb-addr=[::] \
292+
--db-nb-addr=$BIND_LOCAL_ADDR \
293+
--db-sb-addr=$BIND_LOCAL_ADDR \
294+
--db-nb-use-remote-in-db=no \
295+
--db-sb-use-remote-in-db=no \
282296
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
283297
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
284298
start_northd
285-
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":[::]
299+
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":["${DB_NB_ADDR}"]
286300
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000
287301
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set NB_Global . options:use_logical_dp_groups=true
288302

289-
ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":[::]
303+
ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":["${DB_SB_ADDR}"]
290304
ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000
291305
else
292306
# get leader if cluster exists
@@ -328,8 +342,10 @@ else
328342
--db-sb-cluster-local-addr="[${POD_IP}]" \
329343
--db-nb-cluster-remote-addr="[${nb_leader_ip}]" \
330344
--db-sb-cluster-remote-addr="[${sb_leader_ip}]" \
331-
--db-nb-addr=[::] \
332-
--db-sb-addr=[::] \
345+
--db-nb-addr=$BIND_LOCAL_ADDR \
346+
--db-sb-addr=$BIND_LOCAL_ADDR \
347+
--db-nb-use-remote-in-db=no \
348+
--db-sb-use-remote-in-db=no \
333349
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
334350
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
335351
start_northd

0 commit comments

Comments
 (0)