Makefile: add deepflow and kwok installation (#3036)

Author: zhangzujian

.github/workflows/build-windows.yaml

@@ -23,7 +23,7 @@ concurrency:
 
 env:
   GO_VERSION: '1.20.5'
-  GOSEC_VERSION: '2.15.0'
+  GOSEC_VERSION: '2.16.0'
 
 jobs:
   filter:

.github/workflows/build-x86-image.yaml

@@ -22,9 +22,9 @@ concurrency:
 
 env:
   GO_VERSION: '1.20.5'
-  GOSEC_VERSION: '2.15.0'
-  HELM_VERSION: v3.11.3
-  SUBMARINER_VERSION: '0.14.3'
+  GOSEC_VERSION: '2.16.0'
+  HELM_VERSION: v3.12.2
+  SUBMARINER_VERSION: '0.14.6'
 
 jobs:
   build-kube-ovn-base:

.github/workflows/scheduled-e2e.yaml

@@ -11,8 +11,8 @@ concurrency:
 
 env:
   GO_VERSION: '1.20.5'
-  HELM_VERSION: v3.11.3
-  SUBMARINER_VERSION: '0.14.3'
+  HELM_VERSION: v3.12.2
+  SUBMARINER_VERSION: '0.14.6'
 
 jobs:
   k8s-conformance-e2e:

.gitignore

@@ -15,6 +15,9 @@ ovn.yaml
 ovn-ic-0.yaml
 ovn-ic-1.yaml
 kind.yaml
+kustomization.yaml
+kwok.yaml
+kwok-node.yaml
 broker-info.subm
 broker-info.subm.*
 broker-info-internal.subm

Makefile

@@ -25,7 +25,7 @@ CHART_UPGRADE_RESTART_OVS=$(shell echo $${CHART_UPGRADE_RESTART_OVS:-false})
 MULTUS_IMAGE = ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
 MULTUS_YAML = https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml
 
-KUBEVIRT_VERSION = v0.58.1
+KUBEVIRT_VERSION = v0.59.2
 KUBEVIRT_OPERATOR_IMAGE = quay.io/kubevirt/virt-operator:$(KUBEVIRT_VERSION)
 KUBEVIRT_API_IMAGE = quay.io/kubevirt/virt-api:$(KUBEVIRT_VERSION)
 KUBEVIRT_CONTROLLER_IMAGE = quay.io/kubevirt/virt-controller:$(KUBEVIRT_VERSION)
@@ -36,23 +36,31 @@ KUBEVIRT_OPERATOR_YAML = https://github.com/kubevirt/kubevirt/releases/download/
 KUBEVIRT_CR_YAML = https://github.com/kubevirt/kubevirt/releases/download/$(KUBEVIRT_VERSION)/kubevirt-cr.yaml
 KUBEVIRT_TEST_YAML = https://kubevirt.io/labs/manifests/vm.yaml
 
-CILIUM_VERSION = 1.12.9
+CILIUM_VERSION = 1.13.4
 CILIUM_IMAGE_REPO = quay.io/cilium/cilium
 
-CERT_MANAGER_VERSION = v1.11.1
+CERT_MANAGER_VERSION = v1.12.2
 CERT_MANAGER_CONTROLLER = quay.io/jetstack/cert-manager-controller:$(CERT_MANAGER_VERSION)
 CERT_MANAGER_CAINJECTOR = quay.io/jetstack/cert-manager-cainjector:$(CERT_MANAGER_VERSION)
 CERT_MANAGER_WEBHOOK = quay.io/jetstack/cert-manager-webhook:$(CERT_MANAGER_VERSION)
 CERT_MANAGER_YAML = https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml
 
-SUBMARINER_VERSION = $(shell echo $${SUBMARINER_VERSION:-0.14.3})
+SUBMARINER_VERSION = $(shell echo $${SUBMARINER_VERSION:-0.14.6})
 SUBMARINER_OPERATOR = quay.io/submariner/submariner-operator:$(SUBMARINER_VERSION)
 SUBMARINER_GATEWAY = quay.io/submariner/submariner-gateway:$(SUBMARINER_VERSION)
 SUBMARINER_LIGHTHOUSE_AGENT = quay.io/submariner/lighthouse-agent:$(SUBMARINER_VERSION)
 SUBMARINER_LIGHTHOUSE_COREDNS = quay.io/submariner/lighthouse-coredns:$(SUBMARINER_VERSION)
 SUBMARINER_ROUTE_AGENT = quay.io/submariner/submariner-route-agent:$(SUBMARINER_VERSION)
 SUBMARINER_NETTEST = quay.io/submariner/nettest:$(SUBMARINER_VERSION)
 
+DEEPFLOW_CHART_VERSION = 6.2.6
+DEEPFLOW_CHART_REPO = https://deepflow-ce.oss-cn-beijing.aliyuncs.com/chart/stable
+DEEPFLOW_IMAGE_REPO = registry.cn-beijing.aliyuncs.com/deepflow-ce
+DEEPFLOW_GRAFANA_PORT = 30080
+
+KWOK_VERSION = v0.3.0
+KWOK_IMAGE = registry.k8s.io/kwok/kwok:$(KWOK_VERSION)
+
 VPC_NAT_GW_IMG = $(REGISTRY)/vpc-nat-gateway:$(VERSION)
 
 E2E_NETWORK = bridge
@@ -245,8 +253,10 @@ endef
 
 define kind_create_cluster
 	kind create cluster --config $(1) --name $(2)
-	kubectl delete --ignore-not-found sc standard
-	kubectl delete --ignore-not-found -n local-path-storage deploy local-path-provisioner
+	@if [ "x$(3)" = "x1" ]; then \
+		kubectl delete --ignore-not-found sc standard; \
+		kubectl delete --ignore-not-found -n local-path-storage deploy local-path-provisioner; \
+	fi
 	kubectl describe no
 endef
 
@@ -266,6 +276,10 @@ define kind_load_submariner_images
 	$(call kind_load_image,$(1),$(SUBMARINER_NETTEST),1)
 endef
 
+define kind_load_kwok_image
+	$(call kind_load_image,$(1),$(KWOK_IMAGE),1)
+endef
+
 define kubectl_wait_exist_and_ready
 	@echo "Waiting for $(2) $(1)/$(3) to exist..."
 	@n=0; while ! kubectl -n $(1) get $(2) -o name | awk -F / '{print $$2}' | grep -q ^$(3)$$; do \
@@ -317,7 +331,7 @@ kind-enable-hairpin:
 
 .PHONY: kind-create
 kind-create:
-	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn)
+	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn,1)
 
 .PHONY: kind-init
 kind-init: kind-init-ipv4
@@ -333,22 +347,27 @@ kind-init-ovn-ic: kind-init-ovn-ic-ipv4
 .PHONY: kind-init-ovn-ic-ipv4
 kind-init-ovn-ic-ipv4: kind-clean-ovn-ic kind-init
 	@$(MAKE) kind-generate-config
-	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)
+	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1)
 
 .PHONY: kind-init-ovn-ic-ipv6
 kind-init-ovn-ic-ipv6: kind-clean-ovn-ic kind-init-ipv6
 	@ip_family=ipv6 $(MAKE) kind-generate-config
-	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)
+	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1)
 
 .PHONY: kind-init-ovn-ic-dual
 kind-init-ovn-ic-dual: kind-clean-ovn-ic kind-init-dual
 	@ip_family=dual $(MAKE) kind-generate-config
-	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)
+	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1)
 
 .PHONY: kind-init-ovn-submariner
 kind-init-ovn-submariner: kind-clean-ovn-submariner kind-init
 	@pod_cidr_v4=10.18.0.0/16 svc_cidr_v4=10.112.0.0/12 $(MAKE) kind-generate-config
-	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)
+	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1)
+
+.PHONY: kind-init-deepflow
+kind-init-deepflow: kind-clean
+	@port_mapping=$(DEEPFLOW_GRAFANA_PORT):$(DEEPFLOW_GRAFANA_PORT) $(MAKE) kind-generate-config
+	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn,0)
 
 .PHONY: kind-init-iptables
 kind-init-iptables:
@@ -510,7 +529,7 @@ kind-install-ovn-ic-dual: kind-install-dual
 	sed -e 's/10.16.0/10.18.0/g' \
 		-e 's/10.96.0/10.98.0/g' \
 		-e 's/100.64.0/100.68.0/g' \
-	    -e 's/fd00:10:16:/fd00:10:18:/g' \
+		-e 's/fd00:10:16:/fd00:10:18:/g' \
 		-e 's/fd00:10:96:/fd00:10:98:/g' \
 		-e 's/fd00:100:64:/fd00:100:68:/g' \
 		-e 's/VERSION=.*/VERSION=$(VERSION)/' \
@@ -745,6 +764,34 @@ kind-install-cilium-chaining: kind-load-image kind-untaint-control-plane
 		ENABLE_LB=false ENABLE_NP=false CNI_CONFIG_PRIORITY=10 bash
 	kubectl describe no
 
+.PHONY: kind-install-deepflow
+kind-install-deepflow: kind-install
+	helm repo add deepflow $(DEEPFLOW_CHART_REPO)
+	helm repo update deepflow
+	$(eval CLICKHOUSE_PERSISTENCE = $(shell helm show values --version $(DEEPFLOW_CHART_VERSION) --jsonpath '{.clickhouse.storageConfig.persistence}' deepflow/deepflow | sed 's/0Gi/Gi/g'))
+	helm install deepflow -n deepflow deepflow/deepflow \
+		--create-namespace --version $(DEEPFLOW_CHART_VERSION) \
+		--set global.image.repository=$(DEEPFLOW_IMAGE_REPO) \
+		--set grafana.image.repository=$(DEEPFLOW_IMAGE_REPO)/grafana \
+		--set deepflow-agent.sysctlInitContainer.enabled=false \
+		--set 'mysql.storageConfig.persistence.size=5Gi' \
+		--set-json 'clickhouse.storageConfig.persistence=$(CLICKHOUSE_PERSISTENCE)'
+	kubectl -n deepflow patch svc deepflow-grafana --type=json \
+		-p '[{"op": "replace", "path": "/spec/ports/0/nodePort", "value": $(DEEPFLOW_GRAFANA_PORT)}]'
+	echo -e "\nGrafana URL: http://127.0.0.1:$(DEEPFLOW_GRAFANA_PORT)\nGrafana auth: admin:deepflow\n"
+
+.PHONY: kind-install-kwok
+kind-install-kwok: kind-install-underlay
+	kwok_version=$(KWOK_VERSION) j2 yamls/kwok-kustomization.yaml.j2 -o kustomization.yaml
+	kubectl kustomize ./ > kwok.yaml
+	$(call kind_load_kwok_image,kube-ovn)
+	kubectl apply -f kwok.yaml
+	kubectl -n kube-system rollout status deploy kwok-controller --timeout 60s
+	for i in {1..20}; do \
+		kwok_node_name=fake-node-$$i j2 yamls/kwok-node.yaml.j2 -o kwok-node.yaml; \
+		kubectl apply -f kwok-node.yaml; \
+	done
+
 .PHONY: kind-reload
 kind-reload: kind-reload-ovs
 	kubectl delete pod -n kube-system -l app=kube-ovn-controller
@@ -809,6 +856,7 @@ clean:
 	$(RM) yamls/kind.yaml
 	$(RM) ovn.yaml kube-ovn.yaml kube-ovn-crd.yaml
 	$(RM) ovn-ic-0.yaml ovn-ic-1.yaml
+	$(RM) kustomization.yaml kwok.yaml kwok-node.yaml
 	$(RM) kube-ovn.tar vpc-nat-gateway.tar image-amd64.tar image-arm64.tar
 
 .PHONY: changelog

yamls/kind.yaml.j2

@@ -55,6 +55,19 @@ kubeadmConfigPatches:
 nodes:
   - role: control-plane
     image: kindest/node:{{ k8s_version }}
+{%- if port_mapping is defined %}
+  {%- set container_port = port_mapping.split(':')[0] %}
+  {%- set host_port = port_mapping.split(':')[1] %}
+    extraPortMappings:
+      - containerPort: {{ container_port }}
+        hostPort: {{ host_port }}
+        # optional: set the bind address on the host
+        # 0.0.0.0 is the current default
+        listenAddress: "127.0.0.1"
+        # optional: set the protocol to one of TCP, UDP, SCTP.
+        # TCP is the default
+        protocol: TCP
+{%- endif %}
 {%- if single is equalto "false" %}
   - role: worker
     image: kindest/node:{{ k8s_version }}

yamls/kwok-kustomization.yaml.j2

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: registry.k8s.io/kwok/kwok
+  newTag: "{{ kwok_version }}"
+resources:
+- "https://github.com/kubernetes-sigs/kwok/kustomize/kwok?ref={{ kwok_version }}"

yamls/kwok-node.yaml.j2

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Node
+metadata:
+  annotations:
+    node.alpha.kubernetes.io/ttl: "0"
+    kwok.x-k8s.io/node: fake
+  labels:
+    beta.kubernetes.io/arch: amd64
+    beta.kubernetes.io/os: linux
+    kubernetes.io/arch: amd64
+    kubernetes.io/hostname: "{{ kwok_node_name }}"
+    kubernetes.io/os: linux
+    kubernetes.io/role: agent
+    node-role.kubernetes.io/agent: ""
+    type: kwok
+  name: "{{ kwok_node_name }}"
+spec:
+  taints: # Avoid scheduling actual running pods to fake Node
+  - effect: NoSchedule
+    key: kwok.x-k8s.io/node
+    value: fake
+status:
+  allocatable:
+    cpu: 32
+    memory: 256Gi
+    pods: 110
+  capacity:
+    cpu: 32
+    memory: 256Gi
+    pods: 110
+  nodeInfo:
+    architecture: amd64
+    bootID: ""
+    containerRuntimeVersion: ""
+    kernelVersion: ""
+    kubeProxyVersion: fake
+    kubeletVersion: fake
+    machineID: ""
+    operatingSystem: linux
+    osImage: ""
+    systemUUID: ""
+  phase: Running