feat: Update oss charts (#15719)

* fix: revert extraEnv delition in values.yaml for bootloader * add newline * feat: Update bootloader,webapp,server. Add way of defining secrets, update extraEnv usage. Add PodDistributionBudget into all deployments * feat: Update oss charts, make them able to be ingested in cloud deployment * fix: include #15685 changes * fix: Update Chart.yaml. fix minio deployment conditional operator * fix: fix EOF in worker, update worker HPA conditional * fix: remove cloud related stuff * fix: add conditional for hooks * fix: remove hooks for worker * fix: update nit, remove gsm * fix: fix nits * fix: remove gsm and hpa from values.yaml
airbytehq · Aug 19, 2022 · 626f51f · 626f51f
1 parent afbe584
commit 626f51f
Show file tree

Hide file tree

Showing 17 changed files with 214 additions and 34 deletions.
diff --git a/charts/airbyte-bootloader/templates/bootloader-secrets.yaml b/charts/airbyte-bootloader/templates/bootloader-secrets.yaml
@@ -0,0 +1,23 @@
+# Create secrets only for the local deployment
+{{- if .Values.secrets }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: bootloader-secrets
+  labels:
+    app.kubernetes.io/name: {{ include "airbyte.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    helm.sh/chart: {{ include "airbyte.chart" . }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.airbyte.io/fullname: {{ include "airbyte.fullname" . }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-weight: "-1"
+
+type: Opaque
+data:
+  {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }}
+    {{ $k }}: {{ if $v }}{{ $v | b64enc  }} {{else}}""{{end}}
+  {{- end }}
+  {{- end }}
diff --git a/charts/airbyte-bootloader/templates/pod.yaml b/charts/airbyte-bootloader/templates/pod.yaml
@@ -71,8 +71,22 @@ spec:
               name: {{ .Values.global.configMapName | default (printf "%s-airbyte-secrets" .Release.Name) }}
               key: DATABASE_USER
         {{- end }}
-        {{- if .Values.extraEnv }}
-        {{ .Values.extraEnv | toYaml | nindent 10 }}
+        # Values from secret
+        {{- if .Values.secrets }}
+        {{- range $k, $v := .Values.secrets }}
+        - name: {{ $k }}
+          valueFrom:
+            secretKeyRef:
+              name: bootloader-secrets
+              key: {{ $k }}
+        {{- end }}
         {{- end }}
 
-      resources: {{- toYaml .Values.resources | nindent 8 }}
+        # Values from env
+        {{- if .Values.extraEnv }}
+        {{- range $k, $v := mergeOverwrite .Values.extraEnv .Values.global.env_vars }}
+        - name: {{ $k }}
+          value: {{ $v | quote }}
+        {{- end }}
+        {{- end }}
+      resources: {{- toYaml .Values.resources | nindent 8 }}
diff --git a/charts/airbyte-bootloader/values.yaml b/charts/airbyte-bootloader/values.yaml
@@ -13,6 +13,8 @@ global:
   database:
     secretName: ""
     secretValue: ""
+  secrets: {}
+  env_vars: {}
 
 
 enabled: true
@@ -61,4 +63,7 @@ resources:
 ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity
 ##
 affinity: {}
-extraEnv: []
+extraEnv: {}
+secrets: {}
+
+
diff --git a/charts/airbyte-server/templates/deployment.yaml b/charts/airbyte-server/templates/deployment.yaml
@@ -130,26 +130,24 @@ spec:
             configMapKeyRef:
               name: {{ .Release.Name }}-airbyte-env
               key: S3_LOG_BUCKET_REGION
-        {{- if and .Values.global.logs.accessKey.existingSecret .Values.global.logs.accessKey.existingSecretKey }}
+        {{- if and .Values.global.logs.accessKey.existingSecret .Values.global.logs.secretKey.existingSecretKey }}
         - name: AWS_ACCESS_KEY_ID
           valueFrom:
             secretKeyRef:
               name: {{ .Values.global.logs.accessKey.existingSecret }}
               key: {{ .Values.global.logs.accessKey.existingSecretKey }}
-        {{- else }}
-        - name: AWS_ACCESS_KEY_ID
-          valueFrom:
-            secretKeyRef:
-              name: {{ .Release.Name }}-minio
-              key: root-user
-        {{- end }}
-        {{- if and .Values.global.logs.secretKey.existingSecret .Values.global.logs.secretKey.existingSecretKey }}
         - name: AWS_SECRET_ACCESS_KEY
           valueFrom:
             secretKeyRef:
               name: {{ .Values.global.logs.secretKey.existingSecret }}
               key: {{ .Values.global.logs.secretKey.existingSecretKey }}
-        {{- else }}
+        {{- end }}
+        {{- if or .Values.global.logs.minio.enabled .Values.global.logs.externalMinio.enabled }}
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-minio
+              key: root-user
         - name: AWS_SECRET_ACCESS_KEY
           valueFrom:
             secretKeyRef:
@@ -187,9 +185,25 @@ spec:
               name: {{ .Release.Name }}-airbyte-env
               key: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION
         {{- end }}
+        # Values from secret
+        {{- if .Values.secrets }}
+        {{- range $k, $v := .Values.secrets }}
+        - name: {{ $k }}
+          valueFrom:
+            secretKeyRef:
+              name: server-secrets
+              key: {{ $k }}
+        {{- end }}
+        {{- end }}
+
+        # Values from env
         {{- if .Values.extraEnv }}
-        {{ .Values.extraEnv | toYaml | nindent 10 }}
+        {{- range $k, $v := mergeOverwrite .Values.extraEnv .Values.global.env_vars }}
+        - name: {{ $k }}
+          value: {{ $v | quote }}
+        {{- end }}
         {{- end }}
+
         {{- if .Values.livenessProbe.enabled }}
         livenessProbe:
           httpGet:
@@ -223,9 +237,12 @@ spec:
         securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }}
         {{- end }}
         volumeMounts:
+        {{- if eq .Values.deploymentMode "oss" }}
         - name: gcs-log-creds-volume
           mountPath: /secrets/gcs-log-creds
           readOnly: true
+        {{- end }}
+
         {{- if .Values.extraVolumeMounts }}
   {{ toYaml .Values.extraVolumeMounts | nindent 8 }}
         {{- end }}
@@ -236,9 +253,11 @@ spec:
       {{ toYaml .Values.global.extraContainers | indent 8 }}
       {{- end }}
       volumes:
+      {{- if eq .Values.deploymentMode "oss" }}
       - name: gcs-log-creds-volume
         secret:
           secretName: {{ ternary (printf "%s-gcs-log-creds" ( .Release.Name )) (.Values.global.credVolumeOverride) (eq .Values.global.deploymentMode "oss") }}
+      {{- end }}
       {{- if .Values.extraVolumes }}
 {{ toYaml .Values.extraVolumes | nindent 6 }}
       {{- end }}
diff --git a/charts/airbyte-server/templates/secrets.yaml b/charts/airbyte-server/templates/secrets.yaml
@@ -0,0 +1,20 @@
+# Create secrets only for the local deployment
+{{- if .Values.secrets }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: server-secrets
+  labels:
+    app.kubernetes.io/name: {{ include "airbyte.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    helm.sh/chart: {{ include "airbyte.chart" . }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.airbyte.io/fullname: {{ include "airbyte.fullname" . }}
+  annotations:
+type: Opaque
+data:
+  {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }}
+  {{ $k }}: {{ if $v }}{{ $v | b64enc  }} {{else}}""{{end}}
+  {{- end }}
+{{- end }}
diff --git a/charts/airbyte-server/templates/service.yaml b/charts/airbyte-server/templates/service.yaml
@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "common.names.fullname" . }}
+  name: airbyte-server-svc
 spec:
   type: {{ .Values.service.type }}
   ports:

diff --git a/charts/airbyte-server/values.yaml b/charts/airbyte-server/values.yaml
@@ -165,7 +165,7 @@ log:
 ## extraEnv:
 ## - name: SAMPLE_ENV_VAR
 ##   value: "key=sample-value"
-extraEnv: []
+extraEnv: {}
 
 ## @param server.extraVolumeMounts [array] Additional volumeMounts for server container(s).
 ## Examples (when using `server.containerSecurityContext.readOnlyRootFilesystem=true`):
@@ -185,4 +185,6 @@ extraVolumes: []
 
 extraContainers: []
 
-extraInitContainers: []
+extraInitContainers: []
+
+secrets: {}
diff --git a/charts/airbyte-webapp/templates/deployment.yaml b/charts/airbyte-webapp/templates/deployment.yaml
@@ -76,8 +76,23 @@ spec:
               name: {{ .Release.Name }}-airbyte-env
               key: INTERNAL_API_HOST
         {{- end }}
+        # Values from secret
+        {{- if .Values.secrets }}
+        {{- range $k, $v := .Values.secrets }}
+        - name: {{ $k }}
+          valueFrom:
+            secretKeyRef:
+              name: webapp-secrets
+              key: {{ $k }}
+        {{- end }}
+        {{- end }}
+
+        # Values from env
         {{- if .Values.extraEnv }}
-        {{ .Values.extraEnv | toYaml | nindent 10 }}
+        {{- range $k, $v := mergeOverwrite .Values.extraEnv .Values.global.env_vars }}
+        - name: {{ $k }}
+          value: {{ $v | quote }}
+        {{- end }}
         {{- end }}
         {{- if .Values.livenessProbe.enabled }}
         livenessProbe:
@@ -114,6 +129,9 @@ spec:
         {{- if .Values.extraVolumeMounts }}
   {{ toYaml .Values.extraVolumeMounts | nindent 8 }}
         {{- end }}
+        {{- if .Values.global.extraVolumeMounts }}
+  {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }}
+        {{- end }}
       {{- if .Values.extraContainers }}
       {{ toYaml .Values.extraContainers | indent 8 }}
       {{- end }}
@@ -123,4 +141,7 @@ spec:
       volumes:
       {{- if .Values.extraVolumes }}
 {{ toYaml .Values.extraVolumes | nindent 6 }}
+      {{- end }}
+      {{- if .Values.global.extraVolumes }}
+{{ toYaml .Values.global.extraVolumes | nindent 6 }}
       {{- end }}
diff --git a/charts/airbyte-webapp/templates/secrets.yaml b/charts/airbyte-webapp/templates/secrets.yaml
@@ -0,0 +1,20 @@
+# Create secrets only for the local deployment
+{{- if .Values.secrets }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: webapp-secrets
+  labels:
+    app.kubernetes.io/name: {{ include "airbyte.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    helm.sh/chart: {{ include "airbyte.chart" . }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.airbyte.io/fullname: {{ include "airbyte.fullname" . }}
+  annotations:
+type: Opaque
+data:
+  {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }}
+  {{ $k }}: {{ if $v }}{{ $v | b64enc  }} {{else}}""{{end}}
+  {{- end }}
+{{- end }}
diff --git a/charts/airbyte-webapp/templates/service.yaml b/charts/airbyte-webapp/templates/service.yaml
@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "common.names.fullname" . }}
+  name: airbyte-webapp-svc
   {{- with .Values.service.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}

diff --git a/charts/airbyte-webapp/values.yaml b/charts/airbyte-webapp/values.yaml
@@ -151,7 +151,7 @@ fullstory:
 ## extraEnv:
 ## - name: SAMPLE_ENV_VAR
 ##   value: "key=sample-value"
-extraEnv: []
+extraEnv: {}
 
 ## @param webapp.extraVolumeMounts [array] Additional volumeMounts for webapp container(s).
 ## Examples (when using `webapp.containerSecurityContext.readOnlyRootFilesystem=true`):
@@ -177,4 +177,6 @@ extraVolumeMounts: []
 ##
 extraVolumes: []
 
-extraContainers: []
+extraContainers: []
+
+secrets: {}
diff --git a/charts/airbyte-worker/templates/deployment.yaml b/charts/airbyte-worker/templates/deployment.yaml
@@ -189,20 +189,20 @@ spec:
             secretKeyRef:
               name: {{ .Values.global.logs.accessKey.existingSecret }}
               key: {{ .Values.global.logs.accessKey.existingSecretKey }}
-        {{- else }}
-        - name: AWS_ACCESS_KEY_ID
-          valueFrom:
-            secretKeyRef:
-              name: {{ .Release.Name }}-minio
-              key: root-user
         {{- end }}
         {{- if and .Values.global.logs.secretKey.existingSecret .Values.global.logs.secretKey.existingSecretKey }}
         - name: AWS_SECRET_ACCESS_KEY
           valueFrom:
             secretKeyRef:
               name: {{ .Values.global.logs.secretKey.existingSecret }}
               key: {{ .Values.global.logs.secretKey.existingSecretKey }}
-        {{- else }}
+        {{- end }}
+        {{- if or .Values.global.logs.minio.enabled .Values.global.logs.externalMinio.enabled }}
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-minio
+              key: root-user
         - name: AWS_SECRET_ACCESS_KEY
           valueFrom:
             secretKeyRef:
@@ -300,9 +300,25 @@ spec:
               name: {{ .Release.Name }}-airbyte-env
               key: USE_STREAM_CAPABLE_STATE
         {{- end }}
+        # Values from secret
+        {{- if .Values.secrets }}
+        {{- range $k, $v := .Values.secrets }}
+        - name: {{ $k }}
+          valueFrom:
+            secretKeyRef:
+              name: worker-secrets
+              key: {{ $k }}
+        {{- end }}
+        {{- end }}
+
+        # Values from env
         {{- if .Values.extraEnv }}
-        {{ .Values.extraEnv | toYaml | nindent 10 }}
+        {{- range $k, $v := mergeOverwrite .Values.extraEnv .Values.global.env_vars }}
+        - name: {{ $k }}
+          value: {{ $v | quote }}
+        {{- end }}
         {{- end }}
+
         {{- if .Values.livenessProbe.enabled }}
         livenessProbe:
           httpGet:
@@ -364,23 +380,33 @@ spec:
         {{- if .Values.containerSecurityContext }}
         securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }}
         {{- end }}
+        {{- if eq .Values.global.deploymentMode "oss" }}
         volumeMounts:
         - name: gcs-log-creds-volume
           mountPath: /secrets/gcs-log-creds
           readOnly: true
+        {{- end }}
         {{- if .Values.extraVolumeMounts }}
 {{ toYaml .Values.extraVolumeMounts | nindent 8 }}
         {{- end }}
+        {{- if .Values.global.extraVolumeMounts }}
+{{ toYaml .Values.global.extraVolumeMounts | nindent 8 }}
+        {{- end }}
       {{- if .Values.extraContainers }}
       {{ toYaml .Values.extraContainers | indent 8 }}
       {{- end }}
       {{- if .Values.global.extraContainers }}
       {{ toYaml .Values.global.extraContainers | indent 8 }}
       {{- end }}
       volumes:
+      {{- if eq .Values.global.deploymentMode "oss" }}
       - name: gcs-log-creds-volume
         secret:
           secretName: {{ ternary (printf "%s-gcs-log-creds" ( .Release.Name )) (.Values.global.credVolumeOverride) (eq .Values.global.deploymentMode "oss") }}
+      {{- end }}
       {{- if .Values.extraVolumes }}
 {{ toYaml .Values.extraVolumes | nindent 6 }}
+      {{- end }}
+      {{- if .Values.global.extraVolumes }}
+{{ toYaml .Values.global.extraVolumes | nindent 6 }}
       {{- end }}