From c2b321d68f8b101783389698164b1c03f2a1c096 Mon Sep 17 00:00:00 2001 From: Ben Stuart Date: Wed, 6 Sep 2023 00:09:18 +0100 Subject: [PATCH 1/6] feat: Add s3 sync sidecar option Signed-off-by: Ben Stuart --- charts/airflow/README.md | 2 +- charts/airflow/templates/NOTES.txt | 6 +- charts/airflow/templates/_helpers/common.tpl | 4 +- charts/airflow/templates/_helpers/pods.tpl | 116 +++++++---- .../templates/_helpers/validate-values.tpl | 31 ++- .../templates/config/secret-config-envs.yaml | 4 +- .../templates/config/secret-known-hosts.yaml | 6 +- .../db-migrations-deployment.yaml | 8 +- .../db-migrations/db-migrations-job.yaml | 4 +- .../templates/flower/flower-deployment.yaml | 8 +- .../scheduler/scheduler-deployment.yaml | 10 +- .../sync/sync-connections-deployment.yaml | 10 +- .../templates/sync/sync-connections-job.yaml | 4 +- .../templates/sync/sync-pools-deployment.yaml | 8 +- .../templates/sync/sync-pools-job.yaml | 4 +- .../templates/sync/sync-users-deployment.yaml | 8 +- .../templates/sync/sync-users-job.yaml | 4 +- .../sync/sync-variables-deployment.yaml | 8 +- .../templates/sync/sync-variables-job.yaml | 4 +- .../triggerer/triggerer-deployment.yaml | 8 +- .../webserver/webserver-deployment.yaml | 8 +- .../templates/worker/worker-statefulset.yaml | 8 +- charts/airflow/values.yaml | 184 +++++++++++------- 23 files changed, 278 insertions(+), 179 deletions(-) diff --git a/charts/airflow/README.md b/charts/airflow/README.md index 9b99229503..f19983f03f 100644 --- a/charts/airflow/README.md +++ b/charts/airflow/README.md @@ -385,7 +385,7 @@ Parameter | Description | Default --- | --- | --- `dags.path` | the airflow dags folder | `/opt/airflow/dags` `dags.persistence.*` | configs for the dags PVC | `` -`dags.gitSync.*` | configs for the git-sync sidecar | `` +`dags.sync.*` | configs for the sync sidecars | `` diff --git a/charts/airflow/templates/NOTES.txt b/charts/airflow/templates/NOTES.txt index 86ca996e5b..e8246976e8 100644 --- a/charts/airflow/templates/NOTES.txt +++ b/charts/airflow/templates/NOTES.txt @@ -104,7 +104,7 @@ {{- /* if we show the git-sync known_hosts warning */ -}} {{- $known_host_warning := false }} -{{- if and (.Values.dags.gitSync.enabled) (.Values.dags.gitSync.sshSecret) (not .Values.dags.gitSync.sshKnownHosts) }} +{{- if and (.Values.dags.sync.enabled) (.Values.dags.sync.git.sshSecret) (not .Values.dags.sync.git.sshKnownHosts) }} {{- $known_host_warning = true }} {{- end }} @@ -212,7 +212,7 @@ Use these commands to port-forward the Services to your localhost: {{- if $known_host_warning }} [HIGH] git-sync ssh known_hosts verification is disabled! - * HELP: set `dags.gitSync.sshKnownHosts` with the ssh fingerprint of your git host + * HELP: set `dags.sync.git.sshKnownHosts` with the ssh fingerprint of your git host {{ end }} {{- if $scheduler_livenessProbe_warning }} @@ -226,4 +226,4 @@ Use these commands to port-forward the Services to your localhost: {{ end }} {{- end }} -======================================================================== \ No newline at end of file +======================================================================== diff --git a/charts/airflow/templates/_helpers/common.tpl b/charts/airflow/templates/_helpers/common.tpl index df2d8ad87b..e548c6b0c8 100644 --- a/charts/airflow/templates/_helpers/common.tpl +++ b/charts/airflow/templates/_helpers/common.tpl @@ -76,8 +76,8 @@ HTTP The path containing DAG files */}} {{- define "airflow.dags.path" -}} -{{- if .Values.dags.gitSync.enabled -}} -{{- printf "%s/repo/%s" (.Values.dags.path | trimSuffix "/") (.Values.dags.gitSync.repoSubPath | trimAll "/") -}} +{{- if and (.Values.dags.sync.enabled) (eq .Values.dags.sync.type "git") -}} +{{- printf "%s/repo/%s" (.Values.dags.path | trimSuffix "/") (.Values.dags.sync.repoSubPath | trimAll "/") -}} {{- else -}} {{- printf .Values.dags.path -}} {{- end -}} diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl index 3cacda28fc..06e5a2866d 100644 --- a/charts/airflow/templates/_helpers/pods.tpl +++ b/charts/airflow/templates/_helpers/pods.tpl @@ -185,26 +185,27 @@ EXAMPLE USAGE: {{ include "airflow.init_container.install_pip_packages" (dict "R {{- end }} {{/* -Define a container which regularly syncs a git-repo -EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") }} +Define a container which regularly syncs from an external service +EXAMPLE USAGE: {{ include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") }} */}} -{{- define "airflow.container.git_sync" }} -{{- if .sync_one_time }} -- name: dags-git-clone -{{- else }} -- name: dags-git-sync +{{- define "airflow.container.sync" }} +- name: dags-{{ .Values.dags.sync.type }}-sync +{{- end }} +{{- with get .Values.dags.sync .Values.dags.sync.type }} + image: {{ .image.repository }}:{{ .image.tag }} {{- end }} - image: {{ .Values.dags.gitSync.image.repository }}:{{ .Values.dags.gitSync.image.tag }} - imagePullPolicy: {{ .Values.dags.gitSync.image.pullPolicy }} + imagePullPolicy: {{ .Values.dags.sync.image.pullPolicy }} securityContext: - runAsUser: {{ .Values.dags.gitSync.image.uid }} - runAsGroup: {{ .Values.dags.gitSync.image.gid }} + runAsUser: {{ .Values.dags.sync.image.uid }} + runAsGroup: {{ .Values.dags.sync.image.gid }} resources: - {{- toYaml .Values.dags.gitSync.resources | nindent 4 }} + {{- toYaml .Values.dags.sync.resources | nindent 4 }} envFrom: {{- include "airflow.envFrom" . | indent 4 }} env: - {{- if .sync_one_time }} + {{- if eq .Values.dags.sync.type "git" }} + {{- with .Values.dags.sync.git }} + {{- if $.sync_one_time }} - name: GIT_SYNC_ONE_TIME value: "true" {{- end }} @@ -213,30 +214,30 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release - name: GIT_SYNC_DEST value: "repo" - name: GIT_SYNC_REPO - value: {{ .Values.dags.gitSync.repo | quote }} + value: {{ .repo | quote }} - name: GIT_SYNC_BRANCH - value: {{ .Values.dags.gitSync.branch | quote }} + value: {{ .branch | quote }} - name: GIT_SYNC_REV - value: {{ .Values.dags.gitSync.revision | quote }} + value: {{ .revision | quote }} - name: GIT_SYNC_DEPTH - value: {{ .Values.dags.gitSync.depth | quote }} + value: {{ .depth | quote }} - name: GIT_SYNC_WAIT - value: {{ .Values.dags.gitSync.syncWait | quote }} + value: {{ $.Values.dags.sync.syncWait | quote }} - name: GIT_SYNC_TIMEOUT - value: {{ .Values.dags.gitSync.syncTimeout | quote }} + value: {{ .syncTimeout | quote }} - name: GIT_SYNC_ADD_USER value: "true" - name: GIT_SYNC_MAX_SYNC_FAILURES - value: {{ .Values.dags.gitSync.maxFailures | quote }} + value: {{ .maxFailures | quote }} - name: GIT_SYNC_SUBMODULES - value: {{ .Values.dags.gitSync.submodules | quote }} - {{- if .Values.dags.gitSync.sshSecret }} + value: {{ .submodules | quote }} + {{- if .sshSecret }} - name: GIT_SYNC_SSH value: "true" - name: GIT_SSH_KEY_FILE value: "/etc/git-secret/id_rsa" {{- end }} - {{- if .Values.dags.gitSync.sshKnownHosts }} + {{- if .sshKnownHosts }} - name: GIT_KNOWN_HOSTS value: "true" - name: GIT_SSH_KNOWN_HOSTS_FILE @@ -245,35 +246,76 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release - name: GIT_KNOWN_HOSTS value: "false" {{- end }} - {{- if .Values.dags.gitSync.httpSecret }} + {{- if .httpSecret }} - name: GIT_SYNC_USERNAME valueFrom: secretKeyRef: - name: {{ .Values.dags.gitSync.httpSecret }} - key: {{ .Values.dags.gitSync.httpSecretUsernameKey }} + name: {{ .httpSecret }} + key: {{ .httpSecretUsernameKey }} - name: GIT_SYNC_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.dags.gitSync.httpSecret }} - key: {{ .Values.dags.gitSync.httpSecretPasswordKey }} + name: {{ .httpSecret }} + key: {{ .httpSecretPasswordKey }} + {{- end }} + {{- end }} + {{- end }} + {{- if eq .Values.dags.sync.type "s3" }} + {{- with .Values.dags.sync.s3 }} + {{- if .secret }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .secret }} + key: {{ .idKey }} + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .secret }} + key: {{ .secretKey }} + {{- end }} + {{- end }} {{- end }} {{- /* this has user-defined variables, so must be included BELOW (so the ABOVE `env` take precedence) */ -}} {{- include "airflow.env" . | indent 4 }} volumeMounts: - name: dags-data mountPath: /dags - {{- if .Values.dags.gitSync.sshSecret }} + {{- if eq .Values.dags.sync.type "git" }} + {{- with .Values.dags.sync.git }} + {{- if .sshSecret }} - name: git-secret mountPath: /etc/git-secret/id_rsa readOnly: true - subPath: {{ .Values.dags.gitSync.sshSecretKey }} + subPath: {{ .sshSecretKey }} {{- end }} - {{- if .Values.dags.gitSync.sshKnownHosts }} + {{- if .sshKnownHosts }} - name: git-known-hosts mountPath: /etc/git-secret/known_hosts readOnly: true subPath: known_hosts {{- end }} + {{- end }} + {{- end }} + {{- if eq .Values.dags.sync.type "s3" }} + {{- with .Values.dags.sync.s3 }} + command: + - "/bin/sh" + - "-c" + {{- if $.sync_one_time }} + - | + aws s3 cp --recursive {{ .s3Path }} /dags + chown -R {{ $.Values.airflow.image.uid }} /dags + {{- else }} + - | + while true; do + aws s3 sync --delete {{ .s3Path }} /dags + chown -R {{ $.Values.airflow.image.uid }} /dags + sleep {{ $.Values.dags.sync.syncWait }} + done + {{- end }} + {{- end }} + {{- end }} {{- end }} {{/* @@ -360,7 +402,7 @@ EXAMPLE USAGE: {{ include "airflow.volumeMounts" (dict "Release" .Release "Value {{- if eq .Values.dags.persistence.accessMode "ReadOnlyMany" }} readOnly: true {{- end }} -{{- else if .Values.dags.gitSync.enabled }} +{{- else if .Values.dags.sync.enabled }} - name: dags-data mountPath: {{ .Values.dags.path }} {{- end }} @@ -423,7 +465,7 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V {{- else }} claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }} {{- end }} -{{- else if .Values.dags.gitSync.enabled }} +{{- else if .Values.dags.sync.enabled }} - name: dags-data emptyDir: {} {{- end }} @@ -447,14 +489,14 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V {{- end }} {{- /* git-sync */ -}} -{{- if .Values.dags.gitSync.enabled }} -{{- if .Values.dags.gitSync.sshSecret }} +{{- if .Values.dags.sync.enabled }} +{{- if .Values.dags.sync.sshSecret }} - name: git-secret secret: - secretName: {{ .Values.dags.gitSync.sshSecret }} + secretName: {{ .Values.dags.sync.sshSecret }} defaultMode: 0644 {{- end }} -{{- if .Values.dags.gitSync.sshKnownHosts }} +{{- if .Values.dags.sync.sshKnownHosts }} - name: git-known-hosts secret: secretName: {{ include "airflow.fullname" . }}-known-hosts diff --git a/charts/airflow/templates/_helpers/validate-values.tpl b/charts/airflow/templates/_helpers/validate-values.tpl index 9ccbd7ffd4..93ff3f3016 100644 --- a/charts/airflow/templates/_helpers/validate-values.tpl +++ b/charts/airflow/templates/_helpers/validate-values.tpl @@ -116,19 +116,30 @@ {{- end }} {{- end }} -{{/* Checks for `dags.gitSync` */}} -{{- if .Values.dags.gitSync.enabled }} +{{/* Checks for `dags.sync` */}} +{{- if .Values.dags.sync.enabled }} {{- if .Values.dags.persistence.enabled }} - {{ required "If `dags.gitSync.enabled=true`, then `persistence.enabled` must be disabled!" nil }} + {{ required "If `dags.sync.enabled=true` and `dags.sync.type='git'`, then `persistence.enabled` must be disabled!" nil }} {{- end }} - {{- if not .Values.dags.gitSync.repo }} - {{ required "If `dags.gitSync.enabled=true`, then `dags.gitSync.repo` must be non-empty!" nil }} + {{- if eq .Values.dags.sync.type "git" }} + {{- with .Values.dags.sync.git }} + {{- if not .repo }} + {{ required "If `dags.sync.enabled=true` and `dags.sync.type='git'`, then `dags.sync.git.repo` must be non-empty!" nil }} + {{- end }} + {{- if and (.sshSecret) (.httpSecret) }} + {{ required "At most, one of `dags.sync.git.sshSecret` and `dags.sync.git.httpSecret` can be defined!" nil }} + {{- end }} + {{- if and (.repo | lower | hasPrefix "git@github.com") (not .sshSecret) }} + {{ required "You must define `dags.sync.git.sshSecret` when using GitHub with SSH for `dags.sync.git.repo`!" nil }} + {{- end }} {{- end }} - {{- if and (.Values.dags.gitSync.sshSecret) (.Values.dags.gitSync.httpSecret) }} - {{ required "At most, one of `dags.gitSync.sshSecret` and `dags.gitSync.httpSecret` can be defined!" nil }} {{- end }} - {{- if and (.Values.dags.gitSync.repo | lower | hasPrefix "git@github.com") (not .Values.dags.gitSync.sshSecret) }} - {{ required "You must define `dags.gitSync.sshSecret` when using GitHub with SSH for `dags.gitSync.repo`!" nil }} + {{- if eq .Values.dags.sync.type "s3" }} + {{- with .Values.dags.sync.s3 }} + {{- if not .s3Path }} + {{ required "If `dags.sync.enabled=true` and `dags.sync.type='s3'`, then `dags.sync.s3.bucket` must be non-empty!" nil }} + {{- end }} + {{- end }} {{- end }} {{- end }} @@ -220,4 +231,4 @@ {{ required "If `externalRedis.host` is set, then `redis.enabled` should be `false`!" nil }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/airflow/templates/config/secret-config-envs.yaml b/charts/airflow/templates/config/secret-config-envs.yaml index e78a1f2e46..ccee5b258a 100644 --- a/charts/airflow/templates/config/secret-config-envs.yaml +++ b/charts/airflow/templates/config/secret-config-envs.yaml @@ -129,9 +129,9 @@ data: AIRFLOW__WEBSERVER__WEB_SERVER_PORT: {{ "8080" | b64enc | quote }} AIRFLOW__CELERY__FLOWER_PORT: {{ "5555" | b64enc | quote }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.airflow.config.AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL) }} + {{- if and (.Values.dags.sync.enabled) (not .Values.airflow.config.AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL) }} ## refresh the dags folder at the same frequency as git-sync - AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: {{ .Values.dags.gitSync.syncWait | toString | b64enc | quote }} + AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: {{ .Values.dags.sync.syncWait | toString | b64enc | quote }} {{- end }} {{- if and (.Values.airflow.legacyCommands) (not .Values.airflow.config.AIRFLOW__WEBSERVER__RBAC) }} diff --git a/charts/airflow/templates/config/secret-known-hosts.yaml b/charts/airflow/templates/config/secret-known-hosts.yaml index 564ba3cdaf..074ad66903 100644 --- a/charts/airflow/templates/config/secret-known-hosts.yaml +++ b/charts/airflow/templates/config/secret-known-hosts.yaml @@ -1,4 +1,4 @@ -{{- if and (.Values.dags.gitSync.enabled) (.Values.dags.gitSync.sshKnownHosts) }} +{{- if and (.Values.dags.sync.enabled) (.Values.dags.sync.git.sshKnownHosts) }} apiVersion: v1 kind: Secret metadata: @@ -9,5 +9,5 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - known_hosts: {{ .Values.dags.gitSync.sshKnownHosts | b64enc | quote }} -{{- end }} \ No newline at end of file + known_hosts: {{ .Values.dags.sync.git.sshKnownHosts | b64enc | quote }} +{{- end }} diff --git a/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml b/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml index 1ad60348ee..e745273aa1 100644 --- a/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml +++ b/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml @@ -82,9 +82,9 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: @@ -107,9 +107,9 @@ spec: - name: scripts mountPath: /mnt/scripts readOnly: true - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" . | indent 8 }} + {{- include "airflow.container.sync" . | indent 8 }} {{- end }} volumes: {{- $volumes | indent 8 }} diff --git a/charts/airflow/templates/db-migrations/db-migrations-job.yaml b/charts/airflow/templates/db-migrations/db-migrations-job.yaml index 2e729af675..cf5c7045b9 100644 --- a/charts/airflow/templates/db-migrations/db-migrations-job.yaml +++ b/charts/airflow/templates/db-migrations/db-migrations-job.yaml @@ -76,9 +76,9 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/flower/flower-deployment.yaml b/charts/airflow/templates/flower/flower-deployment.yaml index 436f7de424..c1a696ed38 100644 --- a/charts/airflow/templates/flower/flower-deployment.yaml +++ b/charts/airflow/templates/flower/flower-deployment.yaml @@ -86,9 +86,9 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -151,9 +151,9 @@ spec: volumeMounts: {{- $volumeMounts | indent 12 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" . | indent 8 }} + {{- include "airflow.container.sync" . | indent 8 }} {{- end }} {{- if .Values.airflow.extraContainers }} {{- toYaml .Values.airflow.extraContainers | nindent 8 }} diff --git a/charts/airflow/templates/scheduler/scheduler-deployment.yaml b/charts/airflow/templates/scheduler/scheduler-deployment.yaml index 9660f83b6a..99f6a5c4d3 100644 --- a/charts/airflow/templates/scheduler/scheduler-deployment.yaml +++ b/charts/airflow/templates/scheduler/scheduler-deployment.yaml @@ -94,8 +94,8 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- if .Values.dags.sync.enabled }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -228,8 +228,8 @@ spec: readOnly: true {{- end }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} - {{- include "airflow.container.git_sync" . | indent 8 }} + {{- if .Values.dags.sync.enabled }} + {{- include "airflow.container.sync" . | indent 8 }} {{- end }} {{- if .Values.scheduler.logCleanup.enabled }} {{- $lc_resources := .Values.scheduler.logCleanup.resources }} @@ -248,4 +248,4 @@ spec: configMap: name: {{ include "airflow.fullname" . }}-pod-template {{- end }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/charts/airflow/templates/sync/sync-connections-deployment.yaml b/charts/airflow/templates/sync/sync-connections-deployment.yaml index 827343056e..5f7d7a7462 100644 --- a/charts/airflow/templates/sync/sync-connections-deployment.yaml +++ b/charts/airflow/templates/sync/sync-connections-deployment.yaml @@ -82,9 +82,9 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -113,9 +113,9 @@ spec: mountPath: "/mnt/templates" readOnly: true {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" . | indent 8 }} + {{- include "airflow.container.sync" . | indent 8 }} {{- end }} volumes: {{- $volumes | indent 8 }} @@ -147,4 +147,4 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/airflow/templates/sync/sync-connections-job.yaml b/charts/airflow/templates/sync/sync-connections-job.yaml index c77002adf8..41928170d8 100644 --- a/charts/airflow/templates/sync/sync-connections-job.yaml +++ b/charts/airflow/templates/sync/sync-connections-job.yaml @@ -76,9 +76,9 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} diff --git a/charts/airflow/templates/sync/sync-pools-deployment.yaml b/charts/airflow/templates/sync/sync-pools-deployment.yaml index 1635a987db..ce667a31ab 100644 --- a/charts/airflow/templates/sync/sync-pools-deployment.yaml +++ b/charts/airflow/templates/sync/sync-pools-deployment.yaml @@ -82,9 +82,9 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -108,9 +108,9 @@ spec: - name: scripts mountPath: /mnt/scripts readOnly: true - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" . | indent 8 }} + {{- include "airflow.container.sync" . | indent 8 }} {{- end }} volumes: {{- $volumes | indent 8 }} diff --git a/charts/airflow/templates/sync/sync-pools-job.yaml b/charts/airflow/templates/sync/sync-pools-job.yaml index cda0f7cdae..78faa8eecf 100644 --- a/charts/airflow/templates/sync/sync-pools-job.yaml +++ b/charts/airflow/templates/sync/sync-pools-job.yaml @@ -76,9 +76,9 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} diff --git a/charts/airflow/templates/sync/sync-users-deployment.yaml b/charts/airflow/templates/sync/sync-users-deployment.yaml index d61d8943dd..6b73e31b52 100644 --- a/charts/airflow/templates/sync/sync-users-deployment.yaml +++ b/charts/airflow/templates/sync/sync-users-deployment.yaml @@ -82,9 +82,9 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -113,9 +113,9 @@ spec: mountPath: "/mnt/templates" readOnly: true {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" . | indent 8 }} + {{- include "airflow.container.sync" . | indent 8 }} {{- end }} volumes: {{- $volumes | indent 8 }} diff --git a/charts/airflow/templates/sync/sync-users-job.yaml b/charts/airflow/templates/sync/sync-users-job.yaml index d96c8da78a..90786c8c44 100644 --- a/charts/airflow/templates/sync/sync-users-job.yaml +++ b/charts/airflow/templates/sync/sync-users-job.yaml @@ -76,9 +76,9 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} diff --git a/charts/airflow/templates/sync/sync-variables-deployment.yaml b/charts/airflow/templates/sync/sync-variables-deployment.yaml index 21926197c9..262c428e15 100644 --- a/charts/airflow/templates/sync/sync-variables-deployment.yaml +++ b/charts/airflow/templates/sync/sync-variables-deployment.yaml @@ -82,9 +82,9 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -113,9 +113,9 @@ spec: mountPath: "/mnt/templates" readOnly: true {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" . | indent 8 }} + {{- include "airflow.container.sync" . | indent 8 }} {{- end }} volumes: {{- $volumes | indent 8 }} diff --git a/charts/airflow/templates/sync/sync-variables-job.yaml b/charts/airflow/templates/sync/sync-variables-job.yaml index bd50349ed5..b21598c61b 100644 --- a/charts/airflow/templates/sync/sync-variables-job.yaml +++ b/charts/airflow/templates/sync/sync-variables-job.yaml @@ -76,9 +76,9 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.sync.enabled }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} diff --git a/charts/airflow/templates/triggerer/triggerer-deployment.yaml b/charts/airflow/templates/triggerer/triggerer-deployment.yaml index 2497f68f43..f0504080ca 100644 --- a/charts/airflow/templates/triggerer/triggerer-deployment.yaml +++ b/charts/airflow/templates/triggerer/triggerer-deployment.yaml @@ -86,8 +86,8 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- if .Values.dags.sync.enabled }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -160,8 +160,8 @@ spec: volumeMounts: {{- $volumeMounts | indent 12 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} - {{- include "airflow.container.git_sync" . | indent 8 }} + {{- if .Values.dags.sync.enabled }} + {{- include "airflow.container.sync" . | indent 8 }} {{- end }} {{- if .Values.airflow.extraContainers }} {{- toYaml .Values.airflow.extraContainers | nindent 8 }} diff --git a/charts/airflow/templates/webserver/webserver-deployment.yaml b/charts/airflow/templates/webserver/webserver-deployment.yaml index 605b012cb3..eea4846f10 100644 --- a/charts/airflow/templates/webserver/webserver-deployment.yaml +++ b/charts/airflow/templates/webserver/webserver-deployment.yaml @@ -88,8 +88,8 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- if .Values.dags.sync.enabled }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -144,8 +144,8 @@ spec: subPath: webserver_config.py readOnly: true {{- end }} - {{- if .Values.dags.gitSync.enabled }} - {{- include "airflow.container.git_sync" . | indent 8 }} + {{- if .Values.dags.sync.enabled }} + {{- include "airflow.container.sync" . | indent 8 }} {{- end }} {{- if .Values.airflow.extraContainers }} {{- toYaml .Values.airflow.extraContainers | nindent 8 }} diff --git a/charts/airflow/templates/worker/worker-statefulset.yaml b/charts/airflow/templates/worker/worker-statefulset.yaml index d8e5e9498c..a06095b81c 100644 --- a/charts/airflow/templates/worker/worker-statefulset.yaml +++ b/charts/airflow/templates/worker/worker-statefulset.yaml @@ -91,8 +91,8 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} + {{- if .Values.dags.sync.enabled }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -206,8 +206,8 @@ spec: volumeMounts: {{- $volumeMounts | indent 12 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} - {{- include "airflow.container.git_sync" . | indent 8 }} + {{- if .Values.dags.sync.enabled }} + {{- include "airflow.container.sync" . | indent 8 }} {{- end }} {{- if .Values.workers.logCleanup.enabled }} {{- $lc_resources := .Values.workers.logCleanup.resources }} diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml index 3712eb0938..3e4275f0b8 100644 --- a/charts/airflow/values.yaml +++ b/charts/airflow/values.yaml @@ -283,7 +283,7 @@ airflow: ## FILE | pod_template.yaml ######################################## ## - generates a file for `AIRFLOW__KUBERNETES__POD_TEMPLATE_FILE` - ## - the `dags.gitSync` values will create a git-sync init-container in the pod + ## - the `dags.sync` values will create a sync init-container in the pod ## - the `airflow.extraPipPackages` will NOT be installed ## kubernetesPodTemplate: @@ -919,7 +919,7 @@ workers: minAvailable: "" ## configs for the HorizontalPodAutoscaler of the worker Pods - ## - [WARNING] if using git-sync, ensure `dags.gitSync.resources` is set + ## - [WARNING] if using sync, ensure `dags.sync.resources` is set ## - [WARNING] if using worker log-cleanup, ensure `workers.logCleanup.resources` is set ## ## ____ EXAMPLE _______________ @@ -1360,18 +1360,21 @@ dags: ## size: 1Gi - ## configs for the git-sync sidecar (https://github.com/kubernetes/git-sync) + ## configs for the sync sidecar ## - gitSync: - ## if the git-sync sidecar container is enabled + sync: + ## if the sync sidecar container is enabled ## enabled: false - ## the git-sync container image + ## type of sync + ## - git (https://github.com/kubernetes/git-sync) + ## - s3 ## + type: null + + ## sync container image image: - repository: registry.k8s.io/git-sync/git-sync - tag: v3.6.5 pullPolicy: IfNotPresent uid: 65533 gid: 65533 @@ -1382,85 +1385,128 @@ dags: ## resources: {} - ## the url of the git repo - ## - ## ____ EXAMPLE _______________ - ## # https git repo - ## repo: "https://github.com/USERNAME/REPOSITORY.git" + ## the number of seconds between syncs ## - ## ____ EXAMPLE _______________ - ## # ssh git repo - ## repo: "git@github.com:USERNAME/REPOSITORY.git" + syncWait: 60 + + ## git sync configs ## - repo: "" + git: + ## the git-sync container image + ## + image: + repository: registry.k8s.io/git-sync/git-sync + tag: v3.6.5 - ## the sub-path within your repo where dags are located - ## - only dags under this path within your repo will be seen by airflow, - ## (note, the full repo will still be cloned) - ## - repoSubPath: "" + ## the url of the git repo + ## + ## ____ EXAMPLE _______________ + ## # https git repo + ## repo: "https://github.com/USERNAME/REPOSITORY.git" + ## + ## ____ EXAMPLE _______________ + ## # ssh git repo + ## repo: "git@github.com:USERNAME/REPOSITORY.git" + ## + repo: "" - ## the git branch to check out - ## - branch: master + ## the sub-path within your repo where dags are located + ## - only dags under this path within your repo will be seen by airflow, + ## (note, the full repo will still be cloned) + ## + repoSubPath: "" - ## the git revision (tag or hash) to check out - ## - revision: HEAD + ## the git branch to check out + ## + branch: master - ## shallow clone with a history truncated to the specified number of commits - ## - depth: 1 + ## the git revision (tag or hash) to check out + ## + revision: HEAD - ## the number of seconds between syncs - ## - syncWait: 60 + ## shallow clone with a history truncated to the specified number of commits + ## + depth: 1 + + ## the max number of seconds allowed for a complete sync + ## + syncTimeout: 120 - ## the max number of seconds allowed for a complete sync - ## - syncTimeout: 120 + ## the git submodule behavior + ## - allowed values: "recursive", "shallow", "off" + ## + submodules: recursive - ## the git submodule behavior - ## - allowed values: "recursive", "shallow", "off" - ## - submodules: recursive + ## the name of a pre-created Secret with git http credentials + ## + httpSecret: "" - ## the name of a pre-created Secret with git http credentials - ## - httpSecret: "" + ## the key in `dags.gitSync.httpSecret` with your git username + ## + httpSecretUsernameKey: username - ## the key in `dags.gitSync.httpSecret` with your git username - ## - httpSecretUsernameKey: username + ## the key in `dags.gitSync.httpSecret` with your git password/token + ## + httpSecretPasswordKey: password - ## the key in `dags.gitSync.httpSecret` with your git password/token - ## - httpSecretPasswordKey: password + ## the name of a pre-created Secret with git ssh credentials + ## + sshSecret: "" - ## the name of a pre-created Secret with git ssh credentials - ## - sshSecret: "" + ## the key in `dags.gitSync.sshSecret` with your ssh-key file + ## + sshSecretKey: id_rsa - ## the key in `dags.gitSync.sshSecret` with your ssh-key file - ## - sshSecretKey: id_rsa + ## the string value of a "known_hosts" file (for SSH only) + ## - [WARNING] known_hosts verification will be disabled if left empty, making you more + ## vulnerable to repo spoofing attacks + ## + ## ____ EXAMPLE _______________ + ## sshKnownHosts: |- + ## ssh-rsa + ## + sshKnownHosts: "" - ## the string value of a "known_hosts" file (for SSH only) - ## - [WARNING] known_hosts verification will be disabled if left empty, making you more - ## vulnerable to repo spoofing attacks - ## - ## ____ EXAMPLE _______________ - ## sshKnownHosts: |- - ## ssh-rsa - ## - sshKnownHosts: "" + ## the number of consecutive failures allowed before aborting + ## - the first sync must succeed + ## - a value of -1 will retry forever after the initial sync + ## + maxFailures: 0 - ## the number of consecutive failures allowed before aborting - ## - the first sync must succeed - ## - a value of -1 will retry forever after the initial sync + ## configs for a sync from s3 object storage ## - maxFailures: 0 + s3: + ## the s3-sync container image + ## + image: + repository: amazon/aws-cli + tag: 2.13.15 + + ## the s3 path to the dags + ## + ## ____ EXAMPLE _______________ + ## # s3 + ## s3Path: "s3://" + ## + ## ____ EXAMPLE _______________ + ## # s3 sub path + ## s3Path: "s3://///" + ## + s3Path: "" + + ## the name of the pre-created Secret with AWS credentials + ## + secret: "" + + ## the key in `dags.sync.git.secret` with your access key id + ## + idKey: keyId + + ## the key in `dags.sync.git.secret` with your secret access key + secretKey: secretKey + + ################################### ## CONFIG | Kubernetes Ingress ################################### From 44a686c6f19de5c8294014353b679959f3669a11 Mon Sep 17 00:00:00 2001 From: Ben Stuart Date: Wed, 6 Sep 2023 00:24:50 +0100 Subject: [PATCH 2/6] fix: add sync type value validation Signed-off-by: Ben Stuart --- charts/airflow/templates/_helpers/common.tpl | 2 +- charts/airflow/templates/_helpers/validate-values.tpl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/airflow/templates/_helpers/common.tpl b/charts/airflow/templates/_helpers/common.tpl index e548c6b0c8..5f28f8626e 100644 --- a/charts/airflow/templates/_helpers/common.tpl +++ b/charts/airflow/templates/_helpers/common.tpl @@ -77,7 +77,7 @@ The path containing DAG files */}} {{- define "airflow.dags.path" -}} {{- if and (.Values.dags.sync.enabled) (eq .Values.dags.sync.type "git") -}} -{{- printf "%s/repo/%s" (.Values.dags.path | trimSuffix "/") (.Values.dags.sync.repoSubPath | trimAll "/") -}} +{{- printf "%s/repo/%s" (.Values.dags.path | trimSuffix "/") (.Values.dags.sync.git.repoSubPath | trimAll "/") -}} {{- else -}} {{- printf .Values.dags.path -}} {{- end -}} diff --git a/charts/airflow/templates/_helpers/validate-values.tpl b/charts/airflow/templates/_helpers/validate-values.tpl index 93ff3f3016..f446a92183 100644 --- a/charts/airflow/templates/_helpers/validate-values.tpl +++ b/charts/airflow/templates/_helpers/validate-values.tpl @@ -118,6 +118,9 @@ {{/* Checks for `dags.sync` */}} {{- if .Values.dags.sync.enabled }} + {{- if not (has .Values.dags.sync.type (list "git" "s3")) }} + {{ required "The `dags.sync.type` must be one of: [git, s3]!" nil }} + {{- end }} {{- if .Values.dags.persistence.enabled }} {{ required "If `dags.sync.enabled=true` and `dags.sync.type='git'`, then `persistence.enabled` must be disabled!" nil }} {{- end }} From 4415a2675608bc0115bfdeed59295e2f2766d6a4 Mon Sep 17 00:00:00 2001 From: Ben Stuart Date: Wed, 6 Sep 2023 00:47:17 +0100 Subject: [PATCH 3/6] fix: linting errors Signed-off-by: Ben Stuart --- charts/airflow/templates/_helpers/pods.tpl | 1 - charts/airflow/values.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl index 06e5a2866d..8876f47f8c 100644 --- a/charts/airflow/templates/_helpers/pods.tpl +++ b/charts/airflow/templates/_helpers/pods.tpl @@ -190,7 +190,6 @@ EXAMPLE USAGE: {{ include "airflow.container.sync" (dict "Release" .Release "Val */}} {{- define "airflow.container.sync" }} - name: dags-{{ .Values.dags.sync.type }}-sync -{{- end }} {{- with get .Values.dags.sync .Values.dags.sync.type }} image: {{ .image.repository }}:{{ .image.tag }} {{- end }} diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml index 3e4275f0b8..a3dce5d3e7 100644 --- a/charts/airflow/values.yaml +++ b/charts/airflow/values.yaml @@ -1371,7 +1371,7 @@ dags: ## - git (https://github.com/kubernetes/git-sync) ## - s3 ## - type: null + type: "" ## sync container image image: From 2aea8e2c475a623409cf66ed50ee12789049c86b Mon Sep 17 00:00:00 2001 From: Ben Stuart Date: Wed, 6 Sep 2023 00:56:47 +0100 Subject: [PATCH 4/6] fix: trailing spaces Signed-off-by: Ben Stuart --- charts/airflow/values.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml index a3dce5d3e7..ce9df54ede 100644 --- a/charts/airflow/values.yaml +++ b/charts/airflow/values.yaml @@ -1360,7 +1360,7 @@ dags: ## size: 1Gi - ## configs for the sync sidecar + ## configs for the sync sidecar ## sync: ## if the sync sidecar container is enabled @@ -1388,7 +1388,7 @@ dags: ## the number of seconds between syncs ## syncWait: 60 - + ## git sync configs ## git: @@ -1481,7 +1481,7 @@ dags: image: repository: amazon/aws-cli tag: 2.13.15 - + ## the s3 path to the dags ## ## ____ EXAMPLE _______________ @@ -1506,7 +1506,6 @@ dags: secretKey: secretKey - ################################### ## CONFIG | Kubernetes Ingress ################################### From 840d798495c9d1ea3de33c549ca3a4d1002201ba Mon Sep 17 00:00:00 2001 From: Ben Stuart Date: Wed, 6 Sep 2023 00:58:52 +0100 Subject: [PATCH 5/6] fix: more trailing spaces Signed-off-by: Ben Stuart --- charts/airflow/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml index ce9df54ede..ba20fe8afb 100644 --- a/charts/airflow/values.yaml +++ b/charts/airflow/values.yaml @@ -1427,7 +1427,7 @@ dags: ## shallow clone with a history truncated to the specified number of commits ## depth: 1 - + ## the max number of seconds allowed for a complete sync ## syncTimeout: 120 From cb9e72e4b52a420d081f0b5d28e2ac12bbbfa72d Mon Sep 17 00:00:00 2001 From: Ben Stuart Date: Wed, 6 Sep 2023 01:26:08 +0100 Subject: [PATCH 6/6] fix: kubernetes pod template using gitSync values Signed-off-by: Ben Stuart --- charts/airflow/files/pod_template.kubernetes-helm-yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/airflow/files/pod_template.kubernetes-helm-yaml b/charts/airflow/files/pod_template.kubernetes-helm-yaml index b4c232542f..b345c56f53 100644 --- a/charts/airflow/files/pod_template.kubernetes-helm-yaml +++ b/charts/airflow/files/pod_template.kubernetes-helm-yaml @@ -43,13 +43,13 @@ spec: securityContext: {{- $podSecurityContext | nindent 4 }} {{- end }} - {{- if or ($extraPipPackages) (.Values.dags.gitSync.enabled) (.Values.airflow.kubernetesPodTemplate.extraInitContainers) }} + {{- if or ($extraPipPackages) (.Values.dags.sync.enabled) (.Values.airflow.kubernetesPodTemplate.extraInitContainers) }} initContainers: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 4 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }} + {{- if .Values.dags.sync.enabled }} + {{- include "airflow.container.sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }} {{- end }} {{- if .Values.airflow.kubernetesPodTemplate.extraInitContainers }} {{- toYaml .Values.airflow.kubernetesPodTemplate.extraInitContainers | nindent 4 }}