Untitled
unknown
yaml
2 years ago
52 kB
6
Indexable
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Default values for airflow.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
# Provide a name to substitute for the full names of resources
fullnameOverride: ""
# Provide a name to substitute for the name of the chart
nameOverride: ""
# Provide a Kubernetes version (used for API Version selection) to override the auto-detected version
kubeVersionOverride: ""
# User and group of airflow user
uid: 50000
gid: 0
# Default security context for airflow
securityContext: {}
# runAsUser: 50000
# fsGroup: 0
# runAsGroup: 0
# Airflow home directory
# Used for mount paths
airflowHome: /opt/airflow
# Default airflow repository -- overrides all the specific images below
defaultAirflowRepository: crlakemodecmdbprd.azurecr.io/airflow-shape
# Default airflow tag to deploy
defaultAirflowTag: "1.0.0"
# Airflow version (Used to make some decisions based on Airflow Version being deployed)
airflowVersion: "2.3.3"
# Images
images:
airflow:
repository: ~
tag: ~
pullPolicy: IfNotPresent
# To avoid images with user code, you can turn this to 'true' and
# all the 'run-airflow-migrations' and 'wait-for-airflow-migrations' containers/jobs
# will use the images from 'defaultAirflowRepository:defaultAirflowTag' values
# to run and wait for DB migrations .
useDefaultImageForMigration: false
# timeout (in seconds) for airflow-migrations to complete
migrationsWaitTimeout: 60
pod_template:
repository: ~
tag: ~
pullPolicy: IfNotPresent
flower:
repository: ~
tag: ~
pullPolicy: IfNotPresent
statsd:
repository: apache/airflow
tag: airflow-statsd-exporter-2021.04.28-v0.17.0
pullPolicy: IfNotPresent
redis:
repository: redis
tag: 6-bullseye
pullPolicy: IfNotPresent
pgbouncer:
repository: apache/airflow
tag: airflow-pgbouncer-2021.04.28-1.14.0
pullPolicy: IfNotPresent
pgbouncerExporter:
repository: apache/airflow
tag: airflow-pgbouncer-exporter-2021.09.22-0.12.0
pullPolicy: IfNotPresent
gitSync:
repository: k8s.gcr.io/git-sync/git-sync
tag: v3.4.0
pullPolicy: IfNotPresent
# Select certain nodes for airflow pods.
nodeSelector: {}
affinity: {}
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
# Add common labels to all objects and pods defined in this chart.
labels: {}
# Ingress configuration
ingress:
# Enable all ingress resources (deprecated - use ingress.web.enabled and ingress.flower.enabled)
enabled: ~
# Configs for the Ingress of the web Service
web:
# Enable web ingress resource
enabled: false
# Annotations for the web Ingress
annotations: {}
# The path for the web Ingress
path: "/"
# The pathType for the above path (used only with Kubernetes v1.19 and above)
pathType: "ImplementationSpecific"
# The hostname for the web Ingress (Deprecated - renamed to `ingress.web.hosts`)
host: ""
# The hostnames or hosts configuration for the web Ingress
hosts: []
# - name: ""
# # configs for web Ingress TLS
# tls:
# # Enable TLS termination for the web Ingress
# enabled: false
# # the name of a pre-created Secret containing a TLS private key and certificate
# secretName: ""
# The Ingress Class for the web Ingress (used only with Kubernetes v1.19 and above)
ingressClassName: ""
# configs for web Ingress TLS (Deprecated - renamed to `ingress.web.hosts[*].tls`)
tls:
# Enable TLS termination for the web Ingress
enabled: false
# the name of a pre-created Secret containing a TLS private key and certificate
secretName: ""
# HTTP paths to add to the web Ingress before the default path
precedingPaths: []
# Http paths to add to the web Ingress after the default path
succeedingPaths: []
# Configs for the Ingress of the flower Service
flower:
# Enable web ingress resource
enabled: false
# Annotations for the flower Ingress
annotations: {}
# The path for the flower Ingress
path: "/"
# The pathType for the above path (used only with Kubernetes v1.19 and above)
pathType: "ImplementationSpecific"
# The hostname for the flower Ingress (Deprecated - renamed to `ingress.flower.hosts`)
host: ""
# The hostnames or hosts configuration for the flower Ingress
hosts: []
# - name: ""
# tls:
# # Enable TLS termination for the flower Ingress
# enabled: false
# # the name of a pre-created Secret containing a TLS private key and certificate
# secretName: ""
# The Ingress Class for the flower Ingress (used only with Kubernetes v1.19 and above)
ingressClassName: ""
# configs for flower Ingress TLS (Deprecated - renamed to `ingress.flower.hosts[*].tls`)
tls:
# Enable TLS termination for the flower Ingress
enabled: false
# the name of a pre-created Secret containing a TLS private key and certificate
secretName: ""
# Network policy configuration
networkPolicies:
# Enabled network policies
enabled: false
# Extra annotations to apply to all
# Airflow pods
airflowPodAnnotations: {}
# Extra annotations to apply to
# main Airflow configmap
airflowConfigAnnotations: {}
# `airflow_local_settings` file as a string (can be templated).
airflowLocalSettings: |-
{{- if semverCompare ">=2.2.0" .Values.airflowVersion }}
{{- if not (or .Values.webserverSecretKey .Values.webserverSecretKeySecretName) }}
from airflow.www.utils import UIAlert
DASHBOARD_UIALERTS = [
UIAlert(
'Usage of a dynamic webserver secret key detected. We recommend a static webserver secret key instead.'
' See the <a href='
'"https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#webserver-secret-key">'
'Helm Chart Production Guide</a> for more details.',
category="warning",
roles=["Admin"],
html=True,
)
]
{{- end }}
{{- end }}
# Enable RBAC (default on most clusters these days)
rbac:
# Specifies whether RBAC resources should be created
create: true
createSCCRoleBinding: false
# Airflow executor
# One of: LocalExecutor, LocalKubernetesExecutor, CeleryExecutor, KubernetesExecutor, CeleryKubernetesExecutor
executor: "CeleryKubernetesExecutor"
# If this is true and using LocalExecutor/KubernetesExecutor/CeleryKubernetesExecutor, the scheduler's
# service account will have access to communicate with the api-server and launch pods.
# If this is true and using CeleryExecutor/KubernetesExecutor/CeleryKubernetesExecutor, the workers
# will be able to launch pods.
allowPodLaunching: true
# Environment variables for all airflow containers
env:
- name: "AZURE_TENANT_ID"
value: "73d27a24-8a3e-417d-b34c-b2dc4ba5c26f"
- name: "AZURE_CLIENT_ID"
value: "9f8b6c55-9b38-488c-8d8c-a3f324e98586"
- name: "AZURE_CLIENT_SECRET"
value: "vSW8Q~L1OtY.1i4PQt-IDhmP~byO3NlLLM_Y-bKm"
- name: "AIRFLOW__SECRETS__BACKEND"
value: "airflow.providers.microsoft.azure.secrets.key_vault.AzureKeyVaultBackend"
- name: "AIRFLOW__SECRETS__BACKEND_KWARGS"
value: '{"connections_prefix": "airflow-connections", "variables_prefix": "airflow-variables", "vault_url": "https://akv-lake-modec-mdb-prd.vault.azure.net/"}'
# - name: "OPENLINEAGE_URL"
# value: 'https://marquez-dev.shapedigital.com'
# - name: "OPENLINEAGE_NAMESPACE"
# value: "airflow-dev"
# Secrets for all airflow containers
secret: []
# - envName: ""
# secretName: ""
# secretKey: ""
# Enables selected built-in secrets that are set via environment variables by default.
# Those secrets are provided by the Helm Chart secrets by default but in some cases you
# might want to provide some of those variables with _CMD or _SECRET variable, and you should
# in this case disable setting of those variables by setting the relevant configuration to false.
enableBuiltInSecretEnvVars:
AIRFLOW__CORE__FERNET_KEY: true
# For Airflow <2.3, backward compatibility; moved to [database] in 2.3
AIRFLOW__CORE__SQL_ALCHEMY_CONN: true
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: true
AIRFLOW_CONN_AIRFLOW_DB: true
AIRFLOW__WEBSERVER__SECRET_KEY: true
AIRFLOW__CELERY__CELERY_RESULT_BACKEND: true
AIRFLOW__CELERY__RESULT_BACKEND: true
AIRFLOW__CELERY__BROKER_URL: true
AIRFLOW__ELASTICSEARCH__HOST: true
AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST: true
# Extra secrets that will be managed by the chart
# (You can use them with extraEnv or extraEnvFrom or some of the extraVolumes values).
# The format is "key/value" where
# * key (can be templated) is the name of the secret that will be created
# * value: an object with the standard 'data' or 'stringData' key (or both).
# The value associated with those keys must be a string (can be templated)
extraSecrets: {}
# eg:
# extraSecrets:
# '{{ .Release.Name }}-airflow-connections':
# type: 'Opaque'
# data: |
# AIRFLOW_CONN_GCP: 'base64_encoded_gcp_conn_string'
# AIRFLOW_CONN_AWS: 'base64_encoded_aws_conn_string'
# stringData: |
# AIRFLOW_CONN_OTHER: 'other_conn'
# '{{ .Release.Name }}-other-secret-name-suffix':
# data: |
# ...
# Extra ConfigMaps that will be managed by the chart
# (You can use them with extraEnv or extraEnvFrom or some of the extraVolumes values).
# The format is "key/value" where
# * key (can be templated) is the name of the configmap that will be created
# * value: an object with the standard 'data' key.
# The value associated with this keys must be a string (can be templated)
extraConfigMaps: {}
# eg:
# extraConfigMaps:
# '{{ .Release.Name }}-airflow-variables':
# data: |
# AIRFLOW_VAR_HELLO_MESSAGE: "Hi!"
# AIRFLOW_VAR_KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}"
# Extra env 'items' that will be added to the definition of airflow containers
# a string is expected (can be templated).
# TODO: difference from `env`? This is a templated string. Probably should template `env` and remove this.
extraEnv: ~
# eg:
# extraEnv: |
# - name: AIRFLOW__CORE__LOAD_EXAMPLES
# value: 'True'
# Extra envFrom 'items' that will be added to the definition of airflow containers
# A string is expected (can be templated).
extraEnvFrom: ~
# eg:
# extraEnvFrom: |
# - secretRef:
# name: '{{ .Release.Name }}-airflow-connections'
# - configMapRef:
# name: '{{ .Release.Name }}-airflow-variables'
# Airflow database & redis config
data:
# If secret names are provided, use those secrets
metadataSecretName: ~
resultBackendSecretName: ~
brokerUrlSecretName: ~
# Otherwise pass connection values in
metadataConnection:
user: postgres
pass: Afw@e3450da93d5c!
protocol: postgresql
host: airflow-postgres-prd.postgres.database.azure.com
port: 5432
db: postgres
sslmode: require
# resultBackendConnection defaults to the same database as metadataConnection
resultBackendConnection: ~
# or, you can use a different database
# resultBackendConnection:
# user: postgres
# pass: postgres
# protocol: postgresql
# host: ~
# port: 5432
# db: postgres
# sslmode: disable
# Note: brokerUrl can only be set during install, not upgrade
brokerUrl: ~
# Fernet key settings
# Note: fernetKey can only be set during install, not upgrade
fernetKey: hv4v_NPwhwMGnVy_O2iXeuvRW9s_Aab3n6nh5UU3teU=
fernetKeySecretName: ~
# Flask secret key for Airflow Webserver: `[webserver] secret_key` in airflow.cfg
webserverSecretKey: cca5b041072713d23eeaa95a2671cc0b
webserverSecretKeySecretName: ~
# In order to use kerberos you need to create secret containing the keytab file
# The secret name should follow naming convention of the application where resources are
# name {{ .Release-name }}-<POSTFIX>. In case of the keytab file, the postfix is "kerberos-keytab"
# So if your release is named "my-release" the name of the secret should be "my-release-kerberos-keytab"
#
# The Keytab content should be available in the "kerberos.keytab" key of the secret.
#
# apiVersion: v1
# kind: Secret
# data:
# kerberos.keytab: <base64_encoded keytab file content>
# type: Opaque
#
#
# If you have such keytab file you can do it with similar
#
# kubectl create secret generic {{ .Release.name }}-kerberos-keytab --from-file=kerberos.keytab
#
#
# Alternatively, instead of manually creating the secret, it is possible to specify
# kerberos.keytabBase64Content parameter. This parameter should contain base64 encoded keytab.
#
kerberos:
enabled: false
ccacheMountPath: /var/kerberos-ccache
ccacheFileName: cache
configPath: /etc/krb5.conf
keytabBase64Content: ~
keytabPath: /etc/airflow.keytab
principal: airflow@FOO.COM
reinitFrequency: 3600
config: |
# This is an example config showing how you can use templating and how "example" config
# might look like. It works with the test kerberos server that we are using during integration
# testing at Apache Airflow (see `scripts/ci/docker-compose/integration-kerberos.yml` but in
# order to make it production-ready you must replace it with your own configuration that
# Matches your kerberos deployment. Administrators of your Kerberos instance should
# provide the right configuration.
[logging]
default = "FILE:{{ template "airflow_logs_no_quote" . }}/kerberos_libs.log"
kdc = "FILE:{{ template "airflow_logs_no_quote" . }}/kerberos_kdc.log"
admin_server = "FILE:{{ template "airflow_logs_no_quote" . }}/kadmind.log"
[libdefaults]
default_realm = FOO.COM
ticket_lifetime = 10h
renew_lifetime = 7d
forwardable = true
[realms]
FOO.COM = {
kdc = kdc-server.foo.com
admin_server = admin_server.foo.com
}
# Airflow Worker Config
workers:
# Number of airflow celery workers in StatefulSet
replicas: 1
# Command to use when running Airflow workers (templated).
command: ~
# Args to use when running Airflow workers (templated).
args:
- "bash"
- "-c"
# The format below is necessary to get `helm lint` happy
- |-
exec \
airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "celery worker" "worker" }}
# Update Strategy when worker is deployed as a StatefulSet
updateStrategy: ~
# Update Strategy when worker is deployed as a Deployment
strategy:
rollingUpdate:
maxSurge: "100%"
maxUnavailable: "50%"
# When not set, the values defined in the global securityContext will be used
securityContext: {}
# runAsUser: 50000
# fsGroup: 0
# runAsGroup: 0
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
# The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the release name
name: ~
# Annotations to add to worker kubernetes service account.
annotations: {}
# Allow KEDA autoscaling.
# Persistence.enabled must be set to false to use KEDA.
keda:
enabled: false
namespaceLabels: {}
# How often KEDA polls the airflow DB to report new scale requests to the HPA
pollingInterval: 5
# How many seconds KEDA will wait before scaling to zero.
# Note that HPA has a separate cooldown period for scale-downs
cooldownPeriod: 30
# Minimum number of workers created by keda
minReplicaCount: 0
# Maximum number of workers created by keda
maxReplicaCount: 10
persistence:
# Enable persistent volumes
enabled: true
# Volume size for worker StatefulSet
size: 100Gi
# If using a custom storageClass, pass name ref to all statefulSets here
storageClassName:
# Execute init container to chown log directory.
# This is currently only needed in kind, due to usage
# of local-path provisioner.
fixPermissions: false
# Annotations to add to worker volumes
annotations: {}
kerberosSidecar:
# Enable kerberos sidecar
enabled: false
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# Grace period for tasks to finish after SIGTERM is sent from kubernetes
terminationGracePeriodSeconds: 600
# This setting tells kubernetes that its ok to evict
# when it wants to scale a node down.
safeToEvict: true
# Launch additional containers into worker.
# Note: If used with KubernetesExecutor, you are responsible for signaling sidecars to exit when the main
# container finishes so Airflow can continue the worker shutdown process!
extraContainers: []
# Add additional init containers into workers.
extraInitContainers: []
# Mount additional volumes into worker.
extraVolumes: []
extraVolumeMounts: []
# Select certain nodes for airflow worker pods.
nodeSelector: {}
priorityClassName: ~
affinity: {}
# default worker affinity is:
# podAntiAffinity:
# preferredDuringSchedulingIgnoredDuringExecution:
# - podAffinityTerm:
# labelSelector:
# matchLabels:
# component: worker
# topologyKey: kubernetes.io/hostname
# weight: 100
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
# hostAliases to use in worker pods.
# See:
# https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/
hostAliases: []
# - ip: "127.0.0.2"
# hostnames:
# - "test.hostname.one"
# - ip: "127.0.0.3"
# hostnames:
# - "test.hostname.two"
podAnnotations: {}
logGroomerSidecar:
# Command to use when running the Airflow worker log groomer sidecar (templated).
command: ~
# Args to use when running the Airflow worker log groomer sidecar (templated).
args: ["bash", "/clean-logs"]
# Number of days to retain logs
retentionDays: 15
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# Airflow scheduler settings
scheduler:
# If the scheduler stops heartbeating for 5 minutes (5*60s) kill the
# scheduler and let Kubernetes restart it
livenessProbe:
initialDelaySeconds: 10
timeoutSeconds: 20
failureThreshold: 5
periodSeconds: 60
command: ~
# Airflow 2.0 allows users to run multiple schedulers,
# However this feature is only recommended for MySQL 8+ and Postgres
replicas: 1
# Command to use when running the Airflow scheduler (templated).
command: ~
# Args to use when running the Airflow scheduler (templated).
args: ["bash", "-c", "exec airflow scheduler"]
# Update Strategy when scheduler is deployed as a StatefulSet
# (when using LocalExecutor and workers.persistence)
updateStrategy: ~
# Update Strategy when scheduler is deployed as a Deployment
# (when not using LocalExecutor and workers.persistence)
strategy: ~
# When not set, the values defined in the global securityContext will be used
securityContext: {}
# runAsUser: 50000
# fsGroup: 0
# runAsGroup: 0
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
# The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the release name
name: ~
# Annotations to add to scheduler kubernetes service account.
annotations: {}
# Scheduler pod disruption budget
podDisruptionBudget:
enabled: false
# PDB configuration
config:
maxUnavailable: 1
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# This setting tells kubernetes that its ok to evict
# when it wants to scale a node down.
safeToEvict: true
# Launch additional containers into scheduler.
extraContainers: []
# Add additional init containers into scheduler.
extraInitContainers: []
# Mount additional volumes into scheduler.
extraVolumes: []
extraVolumeMounts: []
# Select certain nodes for airflow scheduler pods.
nodeSelector: {}
affinity: {}
# default scheduler affinity is:
# podAntiAffinity:
# preferredDuringSchedulingIgnoredDuringExecution:
# - podAffinityTerm:
# labelSelector:
# matchLabels:
# component: scheduler
# topologyKey: kubernetes.io/hostname
# weight: 100
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
priorityClassName: ~
podAnnotations: {}
logGroomerSidecar:
# Whether to deploy the Airflow scheduler log groomer sidecar.
enabled: true
# Command to use when running the Airflow scheduler log groomer sidecar (templated).
command: ~
# Args to use when running the Airflow scheduler log groomer sidecar (templated).
args: ["bash", "/clean-logs"]
# Number of days to retain logs
retentionDays: 15
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
waitForMigrations:
# Whether to create init container to wait for db migrations
enabled: true
# Airflow create user job settings
createUserJob:
# Command to use when running the create user job (templated).
command: ~
# Args to use when running the create user job (templated).
args:
- "bash"
- "-c"
# The format below is necessary to get `helm lint` happy
- |-
exec \
airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "users create" "create_user" }} "$@"
- --
- "-r"
- "{{ .Values.webserver.defaultUser.role }}"
- "-u"
- "{{ .Values.webserver.defaultUser.username }}"
- "-e"
- "{{ .Values.webserver.defaultUser.email }}"
- "-f"
- "{{ .Values.webserver.defaultUser.firstName }}"
- "-l"
- "{{ .Values.webserver.defaultUser.lastName }}"
- "-p"
- "{{ .Values.webserver.defaultUser.password }}"
# Annotations on the create user job pod
annotations: {}
# jobAnnotations are annotations on the create user job
jobAnnotations: {}
# When not set, the values defined in the global securityContext will be used
securityContext: {}
# runAsUser: 50000
# fsGroup: 0
# runAsGroup: 0
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
# The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the release name
name: ~
# Annotations to add to create user kubernetes service account.
annotations: {}
# Launch additional containers into user creation job
extraContainers: []
# Mount additional volumes into user creation job
extraVolumes: []
extraVolumeMounts: []
nodeSelector: {}
affinity: {}
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
# In case you need to disable the helm hooks that create the jobs after install.
# Disable this if you are using ArgoCD for example
useHelmHooks: true
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# Airflow database migration job settings
migrateDatabaseJob:
enabled: true
# Command to use when running the migrate database job (templated).
command: ~
# Args to use when running the migrate database job (templated).
args:
- "bash"
- "-c"
# The format below is necessary to get `helm lint` happy
- |-
exec \
airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "db upgrade" "upgradedb" }}
# Annotations on the database migration pod
annotations: {}
# jobAnnotations are annotations on the database migration job
jobAnnotations: {}
# When not set, the values defined in the global securityContext will be used
securityContext: {}
# runAsUser: 50000
# fsGroup: 0
# runAsGroup: 0
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
# The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the release name
name: ~
# Annotations to add to migrate database job kubernetes service account.
annotations: {}
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# Launch additional containers into database migration job
extraContainers: []
# Mount additional volumes into database migration job
extraVolumes: []
extraVolumeMounts: []
nodeSelector: {}
affinity: {}
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
# In case you need to disable the helm hooks that create the jobs after install.
# Disable this if you are using ArgoCD for example
useHelmHooks: true
# Airflow webserver settings
webserver:
allowPodLogReading: true
livenessProbe:
initialDelaySeconds: 15
timeoutSeconds: 30
failureThreshold: 20
periodSeconds: 5
readinessProbe:
initialDelaySeconds: 15
timeoutSeconds: 30
failureThreshold: 20
periodSeconds: 5
# Number of webservers
replicas: 1
# Command to use when running the Airflow webserver (templated).
command: ~
# Args to use when running the Airflow webserver (templated).
args: ["bash", "-c", "exec airflow webserver"]
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
# The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the release name
name: ~
# Annotations to add to webserver kubernetes service account.
annotations: {}
# Webserver pod disruption budget
podDisruptionBudget:
enabled: false
# PDB configuration
config:
maxUnavailable: 1
# Allow overriding Update Strategy for Webserver
strategy: ~
# When not set, the values defined in the global securityContext will be used
securityContext: {}
# runAsUser: 50000
# fsGroup: 0
# runAsGroup: 0
# Additional network policies as needed (Deprecated - renamed to `webserver.networkPolicy.ingress.from`)
extraNetworkPolicies: []
networkPolicy:
ingress:
# Peers for webserver NetworkPolicy ingress
from: []
# Ports for webserver NetworkPolicy ingress (if `from` is set)
ports:
- port: "{{ .Values.ports.airflowUI }}"
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# Create initial user.
defaultUser:
enabled: true
role: Admin
username: admin
email: shape_de@shapedigital.com
firstName: admin
lastName: dataengineering
password: 'shub_admin@^pi<S#q{nK<fvPB'
# Launch additional containers into webserver.
extraContainers: []
# Add additional init containers into webserver.
extraInitContainers: []
# Mount additional volumes into webserver.
extraVolumes: []
extraVolumeMounts: []
# This string (can be templated) will be mounted into the Airflow Webserver as a custom
# webserver_config.py. You can bake a webserver_config.py in to your image instead.
webserverConfig: ~
# webserverConfig: |
# from airflow import configuration as conf
# # The SQLAlchemy connection string.
# SQLALCHEMY_DATABASE_URI = conf.get('database', 'SQL_ALCHEMY_CONN')
# # Flask-WTF flag for CSRF
# CSRF_ENABLED = True
service:
type: ClusterIP
## service annotations
annotations: {}
ports:
- name: airflow-ui
port: "{{ .Values.ports.airflowUI }}"
# To change the port used to access the webserver:
# ports:
# - name: airflow-ui
# port: 80
# targetPort: airflow-ui
# To only expose a sidecar, not the webserver directly:
# ports:
# - name: only_sidecar
# port: 80
# targetPort: 8888
loadBalancerIP: ~
## Limit load balancer source ips to list of CIDRs
# loadBalancerSourceRanges:
# - "10.123.0.0/16"
loadBalancerSourceRanges: []
# Select certain nodes for airflow webserver pods.
nodeSelector: {}
priorityClassName: ~
affinity: {}
# default webserver affinity is:
# podAntiAffinity:
# preferredDuringSchedulingIgnoredDuringExecution:
# - podAffinityTerm:
# labelSelector:
# matchLabels:
# component: webserver
# topologyKey: kubernetes.io/hostname
# weight: 100
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
podAnnotations: {}
waitForMigrations:
# Whether to create init container to wait for db migrations
enabled: true
# Airflow Triggerer Config
triggerer:
enabled: true
# Number of airflow triggerers in the deployment
replicas: 1
# Command to use when running Airflow triggerers (templated).
command: ~
# Args to use when running Airflow triggerer (templated).
args: ["bash", "-c", "exec airflow triggerer"]
# Update Strategy for triggerers
strategy:
rollingUpdate:
maxSurge: "100%"
maxUnavailable: "50%"
# If the triggerer stops heartbeating for 5 minutes (5*60s) kill the
# triggerer and let Kubernetes restart it
livenessProbe:
initialDelaySeconds: 10
timeoutSeconds: 20
failureThreshold: 5
periodSeconds: 60
command: ~
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
# The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the release name
name: ~
# Annotations to add to triggerer kubernetes service account.
annotations: {}
# When not set, the values defined in the global securityContext will be used
securityContext: {}
# runAsUser: 50000
# fsGroup: 0
# runAsGroup: 0
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# Grace period for triggerer to finish after SIGTERM is sent from kubernetes
terminationGracePeriodSeconds: 60
# This setting tells kubernetes that its ok to evict
# when it wants to scale a node down.
safeToEvict: true
# Launch additional containers into triggerer.
extraContainers: []
# Add additional init containers into triggerers.
extraInitContainers: []
# Mount additional volumes into triggerer.
extraVolumes: []
extraVolumeMounts: []
# Select certain nodes for airflow triggerer pods.
nodeSelector: {}
affinity: {}
# default triggerer affinity is:
# podAntiAffinity:
# preferredDuringSchedulingIgnoredDuringExecution:
# - podAffinityTerm:
# labelSelector:
# matchLabels:
# component: triggerer
# topologyKey: kubernetes.io/hostname
# weight: 100
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
priorityClassName: ~
podAnnotations: {}
waitForMigrations:
# Whether to create init container to wait for db migrations
enabled: true
# Flower settings
flower:
# Enable flower.
# If True, and using CeleryExecutor/CeleryKubernetesExecutor, will deploy flower app.
enabled: false
# Command to use when running flower (templated).
command: ~
# Args to use when running flower (templated).
args:
- "bash"
- "-c"
# The format below is necessary to get `helm lint` happy
- |-
exec \
airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "celery flower" "flower" }}
# Additional network policies as needed (Deprecated - renamed to `flower.networkPolicy.ingress.from`)
extraNetworkPolicies: []
networkPolicy:
ingress:
# Peers for flower NetworkPolicy ingress
from: []
# Ports for flower NetworkPolicy ingress (if ingressPeers is set)
ports:
- port: "{{ .Values.ports.flowerUI }}"
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# When not set, the values defined in the global securityContext will be used
securityContext: {}
# runAsUser: 50000
# fsGroup: 0
# runAsGroup: 0
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
# The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the release name
name: ~
# Annotations to add to worker kubernetes service account.
annotations: {}
# A secret containing the connection
secretName: ~
# Else, if username and password are set, create secret from username and password
username: ~
password: ~
service:
type: ClusterIP
## service annotations
annotations: {}
ports:
- name: flower-ui
port: "{{ .Values.ports.flowerUI }}"
# To change the port used to access flower:
# ports:
# - name: flower-ui
# port: 8080
# targetPort: flower-ui
loadBalancerIP: ~
## Limit load balancer source ips to list of CIDRs
# loadBalancerSourceRanges:
# - "10.123.0.0/16"
loadBalancerSourceRanges: []
# Launch additional containers into the flower pods.
extraContainers: []
# Mount additional volumes into the flower pods.
extraVolumes: []
extraVolumeMounts: []
# Select certain nodes for airflow flower pods.
nodeSelector: {}
affinity: {}
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
priorityClassName: ~
podAnnotations: {}
# StatsD settings
statsd:
enabled: true
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
# The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the release name
name: ~
# Annotations to add to worker kubernetes service account.
annotations: {}
uid: 65534
# When not set, `statsd.uid` will be used
securityContext: {}
# runAsUser: 65534
# fsGroup: 0
# runAsGroup: 0
# Additional network policies as needed
extraNetworkPolicies: []
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
service:
extraAnnotations: {}
# Select certain nodes for StatsD pods.
nodeSelector: {}
affinity: {}
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
priorityClassName: ~
# Additional mappings for StatsD exporter.
extraMappings: []
# PgBouncer settings
pgbouncer:
# Enable PgBouncer
enabled: false
# Command to use for PgBouncer(templated).
command: ["pgbouncer", "-u", "nobody", "/etc/pgbouncer/pgbouncer.ini"]
# Args to use for PgBouncer(templated).
args: ~
auth_type: md5
auth_file: /etc/pgbouncer/users.txt
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
# The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the release name
name: ~
# Annotations to add to worker kubernetes service account.
annotations: {}
# Additional network policies as needed
extraNetworkPolicies: []
# Pool sizes
metadataPoolSize: 10
resultBackendPoolSize: 5
# Maximum clients that can connect to PgBouncer (higher = more file descriptors)
maxClientConn: 100
# supply the name of existing secret with pgbouncer.ini and users.txt defined
# you can load them to a k8s secret like the one below
# apiVersion: v1
# kind: Secret
# metadata:
# name: pgbouncer-config-secret
# data:
# pgbouncer.ini: <base64_encoded pgbouncer.ini file content>
# users.txt: <base64_encoded users.txt file content>
# type: Opaque
#
# configSecretName: pgbouncer-config-secret
#
configSecretName: ~
# PgBouncer pod disruption budget
podDisruptionBudget:
enabled: false
# PDB configuration
config:
maxUnavailable: 1
# Limit the resources to PgBouncer.
# When you specify the resource request the k8s scheduler uses this information to decide which node to
# place the Pod on. When you specify a resource limit for a Container, the kubelet enforces those limits so
# that the running container is not allowed to use more of that resource than the limit you set.
# See: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
# Example:
#
# resource:
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
resources: {}
service:
extraAnnotations: {}
# https://www.pgbouncer.org/config.html
verbose: 0
logDisconnections: 0
logConnections: 0
sslmode: "prefer"
ciphers: "normal"
ssl:
ca: ~
cert: ~
key: ~
# Add extra PgBouncer ini configuration in the databases section:
# https://www.pgbouncer.org/config.html#section-databases
extraIniMetadata: ~
extraIniResultBackend: ~
# Add extra general PgBouncer ini configuration: https://www.pgbouncer.org/config.html
extraIni: ~
# Mount additional volumes into pgbouncer.
extraVolumes: []
extraVolumeMounts: []
# Select certain nodes for PgBouncer pods.
nodeSelector: {}
affinity: {}
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
priorityClassName: ~
uid: 65534
metricsExporterSidecar:
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
sslmode: "disable"
# Configuration for the redis provisioned by the chart
redis:
enabled: true
terminationGracePeriodSeconds: 600
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
# The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the release name
name: ~
# Annotations to add to worker kubernetes service account.
annotations: {}
persistence:
# Enable persistent volumes
enabled: true
# Volume size for worker StatefulSet
size: 1Gi
# If using a custom storageClass, pass name ref to all statefulSets here
storageClassName:
# Annotations to add to redis volumes
annotations: {}
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# If set use as redis secret. Make sure to also set data.brokerUrlSecretName value.
passwordSecretName: ~
# Else, if password is set, create secret with it,
# Otherwise a new password will be generated on install
# Note: password can only be set during install, not upgrade.
password: ~
# This setting tells kubernetes that its ok to evict
# when it wants to scale a node down.
safeToEvict: true
# Select certain nodes for redis pods.
nodeSelector: {}
affinity: {}
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
# Set to 0 for backwards-compatiblity
uid: 0
# If not set, `redis.uid` will be used
securityContext: {}
# runAsUser: 999
# runAsGroup: 0
# Auth secret for a private registry
# This is used if pulling airflow images from a private registry
registry:
secretName: ~
# Example:
# connection:
# user: ~
# pass: ~
# host: ~
# email: ~
connection: {}
# Elasticsearch logging configuration
elasticsearch:
# Enable elasticsearch task logging
enabled: false
# A secret containing the connection
secretName: ~
# Or an object representing the connection
# Example:
# connection:
# user: ~
# pass: ~
# host: ~
# port: ~
connection: {}
# All ports used by chart
ports:
flowerUI: 5555
airflowUI: 8080
workerLogs: 8793
redisDB: 6379
statsdIngest: 9125
statsdScrape: 9102
pgbouncer: 6543
pgbouncerScrape: 9127
# Define any ResourceQuotas for namespace
quotas: {}
# Define default/max/min values for pods and containers in namespace
limits: []
# This runs as a CronJob to cleanup old pods.
cleanup:
enabled: false
# Run every 15 minutes
schedule: "*/15 * * * *"
# Command to use when running the cleanup cronjob (templated).
command: ~
# Args to use when running the cleanup cronjob (templated).
args: ["bash", "-c", "exec airflow kubernetes cleanup-pods --namespace={{ .Release.Namespace }}"]
# Select certain nodes for airflow cleanup pods.
nodeSelector: {}
affinity: {}
tolerations: [{
key: "project",
operator: "Equal",
value: "shapehub",
effect: "NoSchedule",
}]
topologySpreadConstraints: []
podAnnotations: {}
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
create: true
# The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the release name
name: ~
# Annotations to add to cleanup cronjob kubernetes service account.
annotations: {}
# When not set, the values defined in the global securityContext will be used
securityContext: {}
# runAsUser: 50000
# runAsGroup: 0
# Configuration for postgresql subchart
# Not recommended for production
postgresql:
enabled: false
postgresqlPassword: postgres
postgresqlUsername: postgres
# Config settings to go into the mounted airflow.cfg
#
# Please note that these values are passed through the `tpl` function, so are
# all subject to being rendered as go templates. If you need to include a
# literal `{{` in a value, it must be expressed like this:
#
# a: '{{ "{{ not a template }}" }}'
#
# Do not set config containing secrets via plain text values, use Env Var or k8s secret object
# yamllint disable rule:line-length
config:
core:
dags_folder: '{{ include "airflow_dags" . }}'
# This is ignored when used with the official Docker image
load_examples: 'False'
executor: '{{ .Values.executor }}'
# For Airflow 1.10, backward compatibility; moved to [logging] in 2.0
colored_console_log: 'True'
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
logging:
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
colored_console_log: 'True'
metrics:
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
statsd_port: 9125
statsd_prefix: airflow
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
webserver:
enable_proxy_fix: 'True'
# For Airflow 1.10
rbac: 'True'
celery:
worker_concurrency: 16
scheduler:
# statsd params included for Airflow 1.10 backward compatibility; moved to [metrics] in 2.0
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
statsd_port: 9125
statsd_prefix: airflow
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
# `run_duration` included for Airflow 1.10 backward compatibility; removed in 2.0.
run_duration: 41460
elasticsearch:
json_format: 'True'
log_id_template: "{dag_id}_{task_id}_{execution_date}_{try_number}"
elasticsearch_configs:
max_retries: 3
timeout: 30
retry_timeout: 'True'
kerberos:
keytab: '{{ .Values.kerberos.keytabPath }}'
reinit_frequency: '{{ .Values.kerberos.reinitFrequency }}'
principal: '{{ .Values.kerberos.principal }}'
ccache: '{{ .Values.kerberos.ccacheMountPath }}/{{ .Values.kerberos.ccacheFileName }}'
celery_kubernetes_executor:
kubernetes_queue: 'kubernetes'
kubernetes:
namespace: '{{ .Release.Namespace }}'
airflow_configmap: '{{ include "airflow_config" . }}'
airflow_local_settings_configmap: '{{ include "airflow_config" . }}'
pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml'
worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}'
worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}'
multi_namespace_mode: '{{ if .Values.multiNamespaceMode }}True{{ else }}False{{ end }}'
# yamllint enable rule:line-length
# Whether Airflow can launch workers and/or pods in multiple namespaces
# If true, it creates ClusterRole/ClusterRolebinding (with access to entire cluster)
multiNamespaceMode: false
# `podTemplate` is a templated string containing the contents of `pod_template_file.yaml` used for
# KubernetesExecutor workers. The default `podTemplate` will use normal `workers` configuration parameters
# (e.g. `workers.resources`). As such, you normally won't need to override this directly, however,
# you can still provide a completely custom `pod_template_file.yaml` if desired.
# If not set, a default one is created using `files/pod-template-file.kubernetes-helm-yaml`.
podTemplate: ~
# The following example is NOT functional, but meant to be illustrative of how you can provide a custom
# `pod_template_file`. You're better off starting with the default in
# `files/pod-template-file.kubernetes-helm-yaml` and modifying from there.
# We will set `priorityClassName` in this example:
# podTemplate: |
# apiVersion: v1
# kind: Pod
# metadata:
# name: dummy-name
# labels:
# tier: airflow
# component: worker
# release: {{ .Release.Name }}
# spec:
# priorityClassName: high-priority
# containers:
# - name: base
# ...
# Git sync
dags:
persistence:
# Enable persistent volume for storing dags
enabled: true
# Volume size for dags
size: 1Gi
# If using a custom storageClass, pass name here
storageClassName:
# access mode of the persistent volume
accessMode: ReadWriteOnce
## the name of an existing PVC to use
existingClaim:
## optional subpath for dag volume mount
subPath: ~
gitSync:
enabled: false
# git repo clone url
# ssh examples ssh://git@github.com/apache/airflow.git
# git@github.com:apache/airflow.git
# https example: https://github.com/apache/airflow.git
repo: git@ssh.dev.azure.com:v3/ShapeDigitalEnv/Shape%20Hub/modec-airflow-hub
branch: master
rev: HEAD
depth: 1
# the number of consecutive failures allowed before aborting
maxFailures: 0
# subpath within the repo where dags are located
# should be "" if dags are at repo root
subPath: ""
# if your repo needs a user name password
# you can load them to a k8s secret like the one below
# ---
# apiVersion: v1
# kind: Secret
# metadata:
# name: git-credentials
# data:
# GIT_SYNC_USERNAME: <base64_encoded_git_username>
# GIT_SYNC_PASSWORD: <base64_encoded_git_password>
# and specify the name of the secret below
#
# credentialsSecret: git-credentials
#
#
# If you are using an ssh clone url, you can load
# the ssh private key to a k8s secret like the one below
# ---
# apiVersion: v1
# kind: Secret
# metadata:
# name: airflow-ssh-secret
# data:
# # key needs to be gitSshKey
# gitSshKey: <base64_encoded_data>
# and specify the name of the secret below
sshKeySecret: airflow-git-ssh-secret-prd
#
# If you are using an ssh private key, you can additionally
# specify the content of your known_hosts file, example:
#
# knownHosts: |
# <host1>,<ip1> <key1>
# <host2>,<ip2> <key2>
# interval between git sync attempts in seconds
wait: 30
containerName: git-sync
uid: 65533
# When not set, the values defined in the global securityContext will be used
securityContext: {}
# runAsUser: 65533
# runAsGroup: 0
extraVolumeMounts: []
env: []
resources: {}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
logs:
persistence:
# Enable persistent volume for storing logs
enabled: true
# Volume size for logs
size: 100Gi
# If using a custom storageClass, pass name here
storageClassName: azurefile-csi
## the name of an existing PVC to use
existingClaim: azurefile