operator: deprecate api-server-port flag

Also, as being deprecated, we should not open this port on all
interfaces since cilium-operator is running in the host network.

For now we will open on both IPv4 and IPv6 localhost addresses since the
user might run the operator on IPv6-only clusters, or vice-versa, we
don't want the operator to not being able from listening on a port on
the IP version not available on the cluster.

Signed-off-by: André Martins <andre@cilium.io>

Author: aanm

Documentation/cmdref/cilium-operator.md

@@ -15,7 +15,6 @@ cilium-operator [flags]
 ### Options
 
 ```
-      --api-server-port uint16                  Port on which the operator should serve API requests (default 9234)
       --aws-client-burst int                    Burst value allowed for the AWS client used by the AWS ENI IPAM (default 4)
       --aws-client-qps float                    Queries per second limit for the AWS client used by the AWS ENI IPAM (default 20)
       --aws-instance-limit-mapping map          Add or overwrite mappings of AWS instance limit in the form of {"AWS instance type": "Maximum Network Interfaces","IPv4 Addresses per Interface","IPv6 Addresses per Interface"}. cli example: --aws-instance-limit-mapping=a1.medium=2,4,4 --aws-instance-limit-mapping=a2.somecustomflavor=4,5,6 configmap example: {"a1.medium": "2,4,4", "a2.somecustomflavor": "4,5,6"} (default map[])
@@ -45,6 +44,7 @@ cilium-operator [flags]
       --kvstore string                          Key-value store type
       --kvstore-opt map                         Key-value store options (default map[])
       --nodes-gc-interval duration              GC interval for nodes store in the kvstore (default 2m0s)
+      --operator-api-serve-addr string          Address to serve API requests (default "localhost:9234")
       --operator-prometheus-serve-addr string   Address to serve Prometheus metrics (default ":6942")
       --synchronize-k8s-nodes                   Synchronize Kubernetes nodes to kvstore and perform CNP GC (default true)
       --synchronize-k8s-services                Synchronize Kubernetes services to kvstore (default true)

Documentation/install/upgrade.rst

@@ -347,6 +347,11 @@ Deprecated cilium-operator options
   endpoint GC can be done with ``cilium-endpoint-gc-interval=0``.
   This old option will be removed in Cilium 1.9
 
+* ``api-server-port``: This option is being deprecated. The API Server address
+  and port can be enabled with ``operator-api-serve-addr=127.0.0.1:9234``
+  or ``operator-api-serve-addr=[::1]:9234`` for IPv6-only clusters.
+  This old option will be removed in Cilium 1.9
+
 Removed options
 ~~~~~~~~~~~~~~~
 

install/kubernetes/cilium/charts/config/templates/configmap.yaml

@@ -325,3 +325,9 @@ data:
   enable-remote-node-identity: {{ .Values.global.remoteNodeIdentity | quote }}
 
   synchronize-k8s-nodes: {{ .Values.global.synchronizeK8sNodes | quote }}
+
+{{- if .Values.global.ipv4.enabled }}
+  operator-api-serve-addr: '127.0.0.1:9234'
+{{- else }}
+  operator-api-serve-addr: '[::1]:9234'
+{{- end }}

install/kubernetes/cilium/charts/operator/templates/deployment.yaml

@@ -93,6 +93,11 @@ spec:
 {{- end }}
         livenessProbe:
           httpGet:
+{{- if .Values.global.ipv4.enabled }}
+            host: '127.0.0.1'
+{{- else }}
+            host: '[::1]'
+{{- end }}
             path: /healthz
             port: 9234
             scheme: HTTP

install/kubernetes/quick-install.yaml

@@ -1,4 +1,18 @@
 ---
+# Source: cilium/charts/agent/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium
+  namespace: kube-system
+---
+# Source: cilium/charts/operator/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium-operator
+  namespace: kube-system
+---
 # Source: cilium/charts/config/templates/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
@@ -129,20 +143,7 @@ data:
   enable-remote-node-identity: "true"
 
   synchronize-k8s-nodes: "true"
----
-# Source: cilium/charts/agent/templates/serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: cilium
-  namespace: kube-system
----
-# Source: cilium/charts/operator/templates/serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: cilium-operator
-  namespace: kube-system
+  operator-api-serve-addr: '127.0.0.1:9234'
 ---
 # Source: cilium/charts/agent/templates/clusterrole.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -603,6 +604,7 @@ spec:
         name: cilium-operator
         livenessProbe:
           httpGet:
+            host: '127.0.0.1'
             path: /healthz
             port: 9234
             scheme: HTTP

operator/api.go

@@ -24,9 +24,7 @@ import (
 )
 
 // startServer starts an api server listening on the given address.
-func startServer(addr string, shutdownSignal <-chan struct{}, allSystemsGo <-chan struct{}) {
-	log.Infof("Starting apiserver on address %s", addr)
-
+func startServer(shutdownSignal <-chan struct{}, allSystemsGo <-chan struct{}, addrs ...string) {
 	http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 		select {
 		// only start serving the real health check once all systems all up and running
@@ -37,16 +35,46 @@ func startServer(addr string, shutdownSignal <-chan struct{}, allSystemsGo <-cha
 		}
 	})
 
-	srv := &http.Server{Addr: addr}
+	errs := make(chan error, 1)
+	nServers := 0
 
-	go func() {
-		<-shutdownSignal
-		if err := srv.Shutdown(context.Background()); err != nil {
-			log.WithError(err).Error("apiserver shutdown")
+	// Since we are opening this on localhost only, we need to make sure
+	// we can open for both v4 and v6 localhost. In case the user is running
+	// v4-only or v6-only.
+	for _, addr := range addrs {
+		if addr == "" {
+			continue
 		}
-	}()
+		nServers++
+		srv := &http.Server{Addr: addr}
+		errCh := make(chan error, 1)
+
+		go func() {
+			err := srv.ListenAndServe()
+			if err != nil {
+				errCh <- err
+				errs <- err
+			}
+		}()
+		go func() {
+			select {
+			case <-shutdownSignal:
+				if err := srv.Shutdown(context.Background()); err != nil {
+					log.WithError(err).Error("apiserver shutdown")
+				}
+			case err := <-errCh:
+				log.Warnf("Unable to start status api: %s", err)
+			}
+		}()
+		log.Infof("Starting apiserver on address %s", addr)
+	}
 
-	log.Fatalf("Unable to start status api: %s", srv.ListenAndServe())
+	for err := range errs {
+		nServers--
+		if nServers == 0 {
+			log.Fatalf("Unable to start status api: %s", err)
+		}
+	}
 }
 
 func healthHandlerOK(w http.ResponseWriter, r *http.Request) {

operator/flags.go

@@ -154,6 +154,9 @@ func init() {
 	flags.String(option.OperatorPrometheusServeAddr, ":6942", "Address to serve Prometheus metrics")
 	option.BindEnv(option.OperatorPrometheusServeAddr)
 
+	flags.String(option.OperatorAPIServeAddr, "localhost:9234", "Address to serve API requests")
+	option.BindEnv(option.OperatorAPIServeAddr)
+
 	flags.Bool(option.SyncK8sServices, true, "Synchronize Kubernetes services to kvstore")
 	option.BindEnv(option.SyncK8sServices)
 
@@ -166,8 +169,9 @@ func init() {
 	flags.Bool(option.Version, false, "Print version information")
 	option.BindEnv(option.Version)
 
-	// TODO: Urgent fix
+	// Deprecated, remove in 1.9
 	flags.Uint16Var(&apiServerPort, "api-server-port", 9234, "Port on which the operator should serve API requests")
+	flags.MarkDeprecated("api-server-port", fmt.Sprintf("Please use %s instead", option.OperatorAPIServeAddr))
 
 	// Deprecated, remove in 1.9
 	flags.StringVar(&metricsAddress, "metrics-address", ":6942", "Address to serve Prometheus metrics")

operator/main.go

@@ -57,6 +57,7 @@ var (
 		},
 	}
 
+	// Deprecated: remove in 1.9
 	apiServerPort  uint16
 	shutdownSignal = make(chan struct{})
 
@@ -104,12 +105,19 @@ func kvstoreEnabled() bool {
 		option.Config.SyncK8sNodes
 }
 
+func getAPIServerAddr() []string {
+	if option.Config.OperatorAPIServeAddr == "" {
+		return []string{fmt.Sprintf("127.0.0.1:%d", apiServerPort), fmt.Sprintf("[::1]:%d", apiServerPort)}
+	}
+	return []string{option.Config.OperatorAPIServeAddr}
+}
+
 func runOperator(cmd *cobra.Command) {
 	logging.SetupLogging([]string{}, map[string]string{}, "cilium-operator", viper.GetBool("debug"))
 
 	log.Infof("Cilium Operator %s", version.Version)
 	k8sInitDone := make(chan struct{})
-	go startServer(fmt.Sprintf(":%d", apiServerPort), shutdownSignal, k8sInitDone)
+	go startServer(shutdownSignal, k8sInitDone, getAPIServerAddr()...)
 
 	if option.Config.EnableMetrics {
 		registerMetrics()

pkg/option/config.go

@@ -379,6 +379,9 @@ const (
 	// OperatorPrometheusServeAddr IP:Port on which to serve prometheus metrics (pass ":Port" to bind on all interfaces, "" is off)
 	OperatorPrometheusServeAddr = "operator-prometheus-serve-addr"
 
+	// OperatorAPIServeAddr IP:Port on which to serve api requests in operator (pass ":Port" to bind on all interfaces, "" is off)
+	OperatorAPIServeAddr = "operator-api-serve-addr"
+
 	// PrometheusServeAddrDeprecated IP:Port on which to serve prometheus metrics (pass ":Port" to bind on all interfaces, "" is off)
 	PrometheusServeAddrDeprecated = "prometheus-serve-addr-deprecated"
 
@@ -1175,6 +1178,7 @@ type DaemonConfig struct {
 	PProf                       bool
 	PrometheusServeAddr         string
 	OperatorPrometheusServeAddr string
+	OperatorAPIServeAddr        string
 	ToFQDNsMinTTL               int
 
 	// ToFQDNsProxyPort is the user-configured global, shared, DNS listen port used
@@ -1922,6 +1926,7 @@ func (c *DaemonConfig) Populate() {
 	c.PrependIptablesChains = viper.GetBool(PrependIptablesChainsName)
 	c.PrometheusServeAddr = getPrometheusServerAddr()
 	c.OperatorPrometheusServeAddr = viper.GetString(OperatorPrometheusServeAddr)
+	c.OperatorAPIServeAddr = viper.GetString(OperatorAPIServeAddr)
 	c.ProxyConnectTimeout = viper.GetInt(ProxyConnectTimeout)
 	c.BlacklistConflictingRoutes = viper.GetBool(BlacklistConflictingRoutes)
 	c.ReadCNIConfiguration = viper.GetString(ReadCNIConfiguration)