feat: add s3-sync sidecar option (breaking values changes)

charts/airflow/README.md

@@ -385,7 +385,7 @@ Parameter | Description | Default
 --- | --- | ---
 `dags.path` | the airflow dags folder | `/opt/airflow/dags`
 `dags.persistence.*` | configs for the dags PVC | `<see values.yaml>`
-`dags.gitSync.*` | configs for the git-sync sidecar  | `<see values.yaml>`
+`dags.sync.*` | configs for the sync sidecars  | `<see values.yaml>`
 
 </details>
 

charts/airflow/files/pod_template.kubernetes-helm-yaml

@@ -43,13 +43,13 @@ spec:
   securityContext:
     {{- $podSecurityContext | nindent 4 }}
   {{- end }}
-  {{- if or ($extraPipPackages) (.Values.dags.gitSync.enabled) (.Values.airflow.kubernetesPodTemplate.extraInitContainers) }}
+  {{- if or ($extraPipPackages) (.Values.dags.sync.enabled) (.Values.airflow.kubernetesPodTemplate.extraInitContainers) }}
   initContainers:
     {{- if $extraPipPackages }}
     {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 4 }}
     {{- end }}
-    {{- if .Values.dags.gitSync.enabled }}
-    {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }}
+    {{- if .Values.dags.sync.enabled }}
+    {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }}
     {{- end }}
     {{- if .Values.airflow.kubernetesPodTemplate.extraInitContainers }}
     {{- toYaml .Values.airflow.kubernetesPodTemplate.extraInitContainers | nindent 4 }}

charts/airflow/templates/NOTES.txt

@@ -104,7 +104,7 @@
 
 {{- /* if we show the git-sync known_hosts warning */ -}}
 {{- $known_host_warning := false }}
-{{- if and (.Values.dags.gitSync.enabled) (.Values.dags.gitSync.sshSecret) (not .Values.dags.gitSync.sshKnownHosts) }}
+{{- if and (.Values.dags.sync.enabled) (.Values.dags.sync.git.sshSecret) (not .Values.dags.sync.git.sshKnownHosts) }}
 {{- $known_host_warning = true }}
 {{- end }}
 
@@ -212,7 +212,7 @@ Use these commands to port-forward the Services to your localhost:
 
 {{- if $known_host_warning }}
 [HIGH] git-sync ssh known_hosts verification is disabled!
-  * HELP: set `dags.gitSync.sshKnownHosts` with the ssh fingerprint of your git host
+  * HELP: set `dags.sync.git.sshKnownHosts` with the ssh fingerprint of your git host
 {{ end }}
 
 {{- if $scheduler_livenessProbe_warning }}
@@ -226,4 +226,4 @@ Use these commands to port-forward the Services to your localhost:
 {{ end }}
 
 {{- end }}
-========================================================================
+========================================================================

charts/airflow/templates/_helpers/common.tpl

@@ -76,8 +76,8 @@ HTTP
 The path containing DAG files
 */}}
 {{- define "airflow.dags.path" -}}
-{{- if .Values.dags.gitSync.enabled -}}
-{{- printf "%s/repo/%s" (.Values.dags.path | trimSuffix "/") (.Values.dags.gitSync.repoSubPath | trimAll "/") -}}
+{{- if and (.Values.dags.sync.enabled) (eq .Values.dags.sync.type "git") -}}
+{{- printf "%s/repo/%s" (.Values.dags.path | trimSuffix "/") (.Values.dags.sync.git.repoSubPath | trimAll "/") -}}
 {{- else -}}
 {{- printf .Values.dags.path -}}
 {{- end -}}

charts/airflow/templates/_helpers/pods.tpl

@@ -185,26 +185,26 @@ EXAMPLE USAGE: {{ include "airflow.init_container.install_pip_packages" (dict "R
 {{- end }}
 
 {{/*
-Define a container which regularly syncs a git-repo
-EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") }}
+Define a container which regularly syncs from an external service
+EXAMPLE USAGE: {{ include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") }}
 */}}
-{{- define "airflow.container.git_sync" }}
-{{- if .sync_one_time }}
-- name: dags-git-clone
-{{- else }}
-- name: dags-git-sync
+{{- define "airflow.container.sync" }}
+- name: dags-{{ .Values.dags.sync.type }}-sync
+{{- with get .Values.dags.sync .Values.dags.sync.type }}
+  image: {{ .image.repository }}:{{ .image.tag }}
 {{- end }}
-  image: {{ .Values.dags.gitSync.image.repository }}:{{ .Values.dags.gitSync.image.tag }}
-  imagePullPolicy: {{ .Values.dags.gitSync.image.pullPolicy }}
+  imagePullPolicy: {{ .Values.dags.sync.image.pullPolicy }}
   securityContext:
-    runAsUser: {{ .Values.dags.gitSync.image.uid }}
-    runAsGroup: {{ .Values.dags.gitSync.image.gid }}
+    runAsUser: {{ .Values.dags.sync.image.uid }}
+    runAsGroup: {{ .Values.dags.sync.image.gid }}
   resources:
-    {{- toYaml .Values.dags.gitSync.resources | nindent 4 }}
+    {{- toYaml .Values.dags.sync.resources | nindent 4 }}
   envFrom:
     {{- include "airflow.envFrom" . | indent 4 }}
   env:
-    {{- if .sync_one_time }}
+    {{- if eq .Values.dags.sync.type "git" }}
+    {{- with .Values.dags.sync.git }}
+    {{- if $.sync_one_time }}
     - name: GIT_SYNC_ONE_TIME
       value: "true"
     {{- end }}
@@ -213,30 +213,30 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release
     - name: GIT_SYNC_DEST
       value: "repo"
     - name: GIT_SYNC_REPO
-      value: {{ .Values.dags.gitSync.repo | quote }}
+      value: {{ .repo | quote }}
     - name: GIT_SYNC_BRANCH
-      value: {{ .Values.dags.gitSync.branch | quote }}
+      value: {{ .branch | quote }}
     - name: GIT_SYNC_REV
-      value: {{ .Values.dags.gitSync.revision | quote }}
+      value: {{ .revision | quote }}
     - name: GIT_SYNC_DEPTH
-      value: {{ .Values.dags.gitSync.depth | quote }}
+      value: {{ .depth | quote }}
     - name: GIT_SYNC_WAIT
-      value: {{ .Values.dags.gitSync.syncWait | quote }}
+      value: {{ $.Values.dags.sync.syncWait | quote }}
     - name: GIT_SYNC_TIMEOUT
-      value: {{ .Values.dags.gitSync.syncTimeout | quote }}
+      value: {{ .syncTimeout | quote }}
     - name: GIT_SYNC_ADD_USER
       value: "true"
     - name: GIT_SYNC_MAX_SYNC_FAILURES
-      value: {{ .Values.dags.gitSync.maxFailures | quote }}
+      value: {{ .maxFailures | quote }}
     - name: GIT_SYNC_SUBMODULES
-      value: {{ .Values.dags.gitSync.submodules | quote }}
-    {{- if .Values.dags.gitSync.sshSecret }}
+      value: {{ .submodules | quote }}
+    {{- if .sshSecret }}
     - name: GIT_SYNC_SSH
       value: "true"
     - name: GIT_SSH_KEY_FILE
       value: "/etc/git-secret/id_rsa"
     {{- end }}
-    {{- if .Values.dags.gitSync.sshKnownHosts }}
+    {{- if .sshKnownHosts }}
     - name: GIT_KNOWN_HOSTS
       value: "true"
     - name: GIT_SSH_KNOWN_HOSTS_FILE
@@ -245,35 +245,76 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release
     - name: GIT_KNOWN_HOSTS
       value: "false"
     {{- end }}
-    {{- if .Values.dags.gitSync.httpSecret }}
+    {{- if .httpSecret }}
     - name: GIT_SYNC_USERNAME
       valueFrom:
         secretKeyRef:
-          name: {{ .Values.dags.gitSync.httpSecret }}
-          key: {{ .Values.dags.gitSync.httpSecretUsernameKey }}
+          name: {{ .httpSecret }}
+          key: {{ .httpSecretUsernameKey }}
     - name: GIT_SYNC_PASSWORD
       valueFrom:
         secretKeyRef:
-          name: {{ .Values.dags.gitSync.httpSecret }}
-          key: {{ .Values.dags.gitSync.httpSecretPasswordKey }}
+          name: {{ .httpSecret }}
+          key: {{ .httpSecretPasswordKey }}
+    {{- end }}
+    {{- end }}
+    {{- end }}
+    {{- if eq .Values.dags.sync.type "s3" }}
+    {{- with .Values.dags.sync.s3 }}
+    {{- if .secret }}
+    - name: AWS_ACCESS_KEY_ID
+      valueFrom:
+        secretKeyRef:
+          name: {{ .secret }}
+          key: {{ .idKey }}
+    - name: AWS_SECRET_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: {{ .secret }}
+          key: {{ .secretKey }}
+    {{- end }}
+    {{- end }}
     {{- end }}
     {{- /* this has user-defined variables, so must be included BELOW (so the ABOVE `env` take precedence) */ -}}
     {{- include "airflow.env" . | indent 4 }}
   volumeMounts:
     - name: dags-data
       mountPath: /dags
-    {{- if .Values.dags.gitSync.sshSecret }}
+    {{- if eq .Values.dags.sync.type "git" }}
+    {{- with .Values.dags.sync.git }}
+    {{- if .sshSecret }}
     - name: git-secret
       mountPath: /etc/git-secret/id_rsa
       readOnly: true
-      subPath: {{ .Values.dags.gitSync.sshSecretKey }}
+      subPath: {{ .sshSecretKey }}
     {{- end }}
-    {{- if .Values.dags.gitSync.sshKnownHosts }}
+    {{- if .sshKnownHosts }}
     - name: git-known-hosts
       mountPath: /etc/git-secret/known_hosts
       readOnly: true
       subPath: known_hosts
     {{- end }}
+    {{- end }}
+    {{- end }}
+  {{- if eq .Values.dags.sync.type "s3" }}
+  {{- with .Values.dags.sync.s3 }}
+  command:
+    - "/bin/sh"
+    - "-c"
+    {{- if $.sync_one_time }}
+    - |
+      aws s3 cp --recursive {{ .s3Path }} /dags
+      chown -R {{ $.Values.airflow.image.uid }} /dags
+    {{- else }}
+    - |
+      while true; do
+          aws s3 sync --delete {{ .s3Path }} /dags
+          chown -R {{ $.Values.airflow.image.uid }} /dags
+          sleep {{ $.Values.dags.sync.syncWait }}
+      done
+    {{- end }}
+  {{- end }}
+  {{- end }}
 {{- end }}
 
 {{/*
@@ -360,7 +401,7 @@ EXAMPLE USAGE: {{ include "airflow.volumeMounts" (dict "Release" .Release "Value
   {{- if eq .Values.dags.persistence.accessMode "ReadOnlyMany" }}
   readOnly: true
   {{- end }}
-{{- else if .Values.dags.gitSync.enabled }}
+{{- else if .Values.dags.sync.enabled }}
 - name: dags-data
   mountPath: {{ .Values.dags.path }}
 {{- end }}
@@ -423,7 +464,7 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V
     {{- else }}
     claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }}
     {{- end }}
-{{- else if .Values.dags.gitSync.enabled }}
+{{- else if .Values.dags.sync.enabled }}
 - name: dags-data
   emptyDir: {}
 {{- end }}
@@ -447,14 +488,14 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V
 {{- end }}
 
 {{- /* git-sync */ -}}
-{{- if .Values.dags.gitSync.enabled }}
-{{- if .Values.dags.gitSync.sshSecret }}
+{{- if .Values.dags.sync.enabled }}
+{{- if .Values.dags.sync.sshSecret }}
 - name: git-secret
   secret:
-    secretName: {{ .Values.dags.gitSync.sshSecret }}
+    secretName: {{ .Values.dags.sync.sshSecret }}
     defaultMode: 0644
 {{- end }}
-{{- if .Values.dags.gitSync.sshKnownHosts }}
+{{- if .Values.dags.sync.sshKnownHosts }}
 - name: git-known-hosts
   secret:
     secretName: {{ include "airflow.fullname" . }}-known-hosts

charts/airflow/templates/_helpers/validate-values.tpl

@@ -116,19 +116,33 @@
   {{- end }}
 {{- end }}
 
-{{/* Checks for `dags.gitSync` */}}
-{{- if .Values.dags.gitSync.enabled }}
+{{/* Checks for `dags.sync` */}}
+{{- if .Values.dags.sync.enabled }}
+  {{- if not (has .Values.dags.sync.type (list "git" "s3")) }}
+  {{ required "The `dags.sync.type` must be one of: [git, s3]!" nil }}
+  {{- end }}
   {{- if .Values.dags.persistence.enabled }}
-  {{ required "If `dags.gitSync.enabled=true`, then `persistence.enabled` must be disabled!" nil }}
+  {{ required "If `dags.sync.enabled=true` and `dags.sync.type='git'`, then `persistence.enabled` must be disabled!" nil }}
   {{- end }}
-  {{- if not .Values.dags.gitSync.repo }}
-  {{ required "If `dags.gitSync.enabled=true`, then `dags.gitSync.repo` must be non-empty!" nil }}
+  {{- if eq .Values.dags.sync.type "git" }}
+  {{- with .Values.dags.sync.git }}
+    {{- if not .repo }}
+    {{ required "If `dags.sync.enabled=true` and `dags.sync.type='git'`, then `dags.sync.git.repo` must be non-empty!" nil }}
+    {{- end }}
+    {{- if and (.sshSecret) (.httpSecret) }}
+    {{ required "At most, one of `dags.sync.git.sshSecret` and `dags.sync.git.httpSecret` can be defined!" nil }}
+    {{- end }}
+    {{- if and (.repo | lower | hasPrefix "git@github.com") (not .sshSecret) }}
+    {{ required "You must define `dags.sync.git.sshSecret` when using GitHub with SSH for `dags.sync.git.repo`!" nil }}
+    {{- end }}
   {{- end }}
-  {{- if and (.Values.dags.gitSync.sshSecret) (.Values.dags.gitSync.httpSecret) }}
-  {{ required "At most, one of `dags.gitSync.sshSecret` and `dags.gitSync.httpSecret` can be defined!" nil }}
   {{- end }}
-  {{- if and (.Values.dags.gitSync.repo | lower | hasPrefix "git@github.com") (not .Values.dags.gitSync.sshSecret) }}
-  {{ required "You must define `dags.gitSync.sshSecret` when using GitHub with SSH for `dags.gitSync.repo`!" nil }}
+  {{- if eq .Values.dags.sync.type "s3" }}
+  {{- with .Values.dags.sync.s3 }}
+    {{- if not .s3Path }}
+    {{ required "If `dags.sync.enabled=true` and `dags.sync.type='s3'`, then `dags.sync.s3.bucket` must be non-empty!" nil }}
+    {{- end }}
+  {{- end }}
   {{- end }}
 {{- end }}
 
@@ -220,4 +234,4 @@
     {{ required "If `externalRedis.host` is set, then `redis.enabled` should be `false`!" nil }}
     {{- end }}
   {{- end }}
-{{- end }}
+{{- end }}

charts/airflow/templates/config/secret-config-envs.yaml

@@ -129,9 +129,9 @@ data:
   AIRFLOW__WEBSERVER__WEB_SERVER_PORT: {{ "8080" | b64enc | quote }}
   AIRFLOW__CELERY__FLOWER_PORT: {{ "5555" | b64enc | quote }}
 
-  {{- if and (.Values.dags.gitSync.enabled) (not .Values.airflow.config.AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL) }}
+  {{- if and (.Values.dags.sync.enabled) (not .Values.airflow.config.AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL) }}
   ## refresh the dags folder at the same frequency as git-sync
-  AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: {{ .Values.dags.gitSync.syncWait | toString | b64enc | quote }}
+  AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: {{ .Values.dags.sync.syncWait | toString | b64enc | quote }}
   {{- end }}
 
   {{- if and (.Values.airflow.legacyCommands) (not .Values.airflow.config.AIRFLOW__WEBSERVER__RBAC) }}

charts/airflow/templates/config/secret-known-hosts.yaml

@@ -1,4 +1,4 @@
-{{- if and (.Values.dags.gitSync.enabled) (.Values.dags.gitSync.sshKnownHosts) }}
+{{- if and (.Values.dags.sync.enabled) (.Values.dags.sync.git.sshKnownHosts) }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -9,5 +9,5 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 data:
-  known_hosts: {{ .Values.dags.gitSync.sshKnownHosts | b64enc | quote }}
-{{- end }}
+  known_hosts: {{ .Values.dags.sync.git.sshKnownHosts | b64enc | quote }}
+{{- end }}

charts/airflow/templates/db-migrations/db-migrations-deployment.yaml

@@ -82,9 +82,9 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if .Values.dags.sync.enabled }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
-        {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
+        {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
       containers:
@@ -107,9 +107,9 @@ spec:
             - name: scripts
               mountPath: /mnt/scripts
               readOnly: true
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if .Values.dags.sync.enabled }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
-        {{- include "airflow.container.git_sync" . | indent 8 }}
+        {{- include "airflow.container.sync" . | indent 8 }}
         {{- end }}
       volumes:
         {{- $volumes | indent 8 }}

charts/airflow/templates/db-migrations/db-migrations-job.yaml

@@ -76,9 +76,9 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if .Values.dags.sync.enabled }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
-        {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
+        {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
       containers: