diff --git a/charts/shipyard/templates/configmap-airflow-etc.yaml b/charts/shipyard/templates/configmap-airflow-etc.yaml index 5d2a8ce4..147fbc5e 100644 --- a/charts/shipyard/templates/configmap-airflow-etc.yaml +++ b/charts/shipyard/templates/configmap-airflow-etc.yaml @@ -18,25 +18,25 @@ # Add endpoint URI lookup for Airflow Web Server {{- if empty .Values.conf.airflow.webserver.base_url -}} -{{- tuple "airflow_web" "internal" "airflow_web" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup"| set .Values.conf.airflow.webserver "base_url" | quote | trunc 0 -}} +{{- tuple "airflow_web" "internal" "airflow_web" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.airflow.webserver "base_url" | quote | trunc 0 -}} {{- end -}} {{- if empty .Values.conf.airflow.cli.endpoint_url -}} -{{- tuple "airflow_web" "internal" "airflow_web" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup"| set .Values.conf.airflow.cli "endpoint_url" | quote | trunc 0 -}} +{{- tuple "airflow_web" "internal" "airflow_web" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.airflow.cli "endpoint_url" | quote | trunc 0 -}} {{- end -}} # Add endpoint URI lookup for Postgresql DB Connection {{- if empty .Values.conf.airflow.core.sql_alchemy_conn -}} -{{- tuple "postgresql_airflow_db" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup"| set .Values.conf.airflow.core "sql_alchemy_conn" | quote | trunc 0 -}} +{{- tuple "postgresql_airflow_db" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.airflow.core "sql_alchemy_conn" | quote | trunc 0 -}} +{{- end -}} + +{{- if empty .Values.conf.airflow.celery.result_backend -}} +{{- tuple "postgresql_airflow_db" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.airflow.celery "result_backend" | quote | trunc 0 -}} {{- end -}} # Add endpoint URI lookup for RabbitMQ Connection {{- if empty .Values.conf.airflow.celery.broker_url -}} -{{- tuple "oslo_messaging" "internal" "user" "amqp" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup"| set .Values.conf.airflow.celery "broker_url" | quote | trunc 0 -}} -{{- end -}} - -{{- if empty .Values.conf.airflow.celery.celery_result_backend -}} -{{- tuple "oslo_messaging" "internal" "user" "amqp" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup"| set .Values.conf.airflow.celery "celery_result_backend" | quote | trunc 0 -}} +{{- tuple "oslo_messaging" "internal" "user" "amqp" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.airflow.celery "broker_url" | quote | trunc 0 -}} {{- end -}} --- diff --git a/charts/shipyard/templates/etc/_airflow.cfg.tpl b/charts/shipyard/templates/etc/_airflow.cfg.tpl index d04bc2b5..ce423fa5 100644 --- a/charts/shipyard/templates/etc/_airflow.cfg.tpl +++ b/charts/shipyard/templates/etc/_airflow.cfg.tpl @@ -18,41 +18,61 @@ {{- define "airflow.conf.airflow_values_skeleton" -}} {{- if not .core -}}{{- set . "core" dict -}}{{- end -}} +{{- if not .cli -}}{{- set . "cli" dict -}}{{- end -}} +{{- if not .api -}}{{- set . "api" dict -}}{{- end -}} +{{- if not .operators -}}{{- set . "operators" dict -}}{{- end -}} {{- if not .webserver -}}{{- set . "webserver" dict -}}{{- end -}} {{- if not .email -}}{{- set . "email" dict -}}{{- end -}} {{- if not .smtp -}}{{- set . "smtp" dict -}}{{- end -}} {{- if not .celery -}}{{- set . "celery" dict -}}{{- end -}} +{{- if not .celery_broker_transport_options -}}{{- set . "celery_broker_transport_options" dict -}}{{- end -}} {{- if not .scheduler -}}{{- set . "scheduler" dict -}}{{- end -}} -{{- if not .mesos -}}{{- set . "mesos" dict -}}{{- end -}} +{{- if not .ldap -}}{{- set . "ldap" dict -}}{{- end -}} +{{- if not .admin -}}{{- set . "admin" dict -}}{{- end -}} {{- end -}} {{- define "airflow.conf.airflow" -}} [core] -# The home folder for airflow, is ~/airflow +# The home folder for airflow, default is ~/airflow {{ if not .core.airflow_home }}#{{ end }}airflow_home = {{ .core.airflow_home | default "" }} # The folder where your airflow pipelines live, most likely a # subfolder in a code repository +# This path must be absolute {{ if not .core.dags_folder }}#{{ end }}dags_folder = {{ .core.dags_folder | default "" }} -# The folder where airflow should store its log files. This location +# The folder where airflow should store its log files +# This path must be absolute {{ if not .core.base_log_folder }}#{{ end }}base_log_folder = {{ .core.base_log_folder | default "" }} # Airflow can store logs remotely in AWS S3 or Google Cloud Storage. Users -# must supply a remote location URL (starting with either 's3://...' or -# 'gs://...') and an Airflow connection id that provides access to the storage -# location. -{{ if not .core.remote_base_log_folder }}#{{ end }}remote_base_log_folder = {{ .core.remote_base_log_folder | default "" }} +# must supply an Airflow connection id that provides access to the storage +# location. If remote_logging is set to true, see UPDATING.md for additional +# configuration requirements. +{{ if not .core.remote_logging }}#{{ end }}remote_logging = {{ .core.remote_logging | default "" }} {{ if not .core.remote_log_conn_id }}#{{ end }}remote_log_conn_id = {{ .core.remote_log_conn_id | default "" }} -# Use server-side encryption for logs stored in S3 {{ if not .core.encrypt_s3_logs }}#{{ end }}encrypt_s3_logs = {{ .core.encrypt_s3_logs | default "" }} -# deprecated option for remote log storage, use remote_base_log_folder instead! -# s3_log_folder = + +# Logging level +{{ if not .core.logging_level }}#{{ end }}logging_level = {{ .core.logging_level | default "" }} + +# Logging class +# Specify the class that will specify the logging configuration +# This class has to be on the python classpath +{{ if not .core.logging_config_class }}#{{ end }}logging_config_class = {{ .core.logging_config_class | default "" }} + +# Log format +{{ if not .core.log_format }}#{{ end }}log_format = {{ .core.log_format | default "" }} +{{ if not .core.simple_log_format }}#{{ end }}simple_log_format = {{ .core.simple_log_format | default "" }} + +# Default timezone in case supplied date times are naive +# can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam) +{{ if not .core.default_timezone }}#{{ end }}default_timezone = {{ .core.default_timezone | default "" }} # The executor class that airflow should use. Choices include -# SequentialExecutor, LocalExecutor, CeleryExecutor +# SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor {{ if not .core.executor }}#{{ end }}executor = {{ .core.executor | default "" }} # The SqlAlchemy connection string to the metadata database. @@ -60,6 +80,9 @@ # their website {{ if not .core.sql_alchemy_conn }}#{{ end }}sql_alchemy_conn = {{ .core.sql_alchemy_conn | default "" }} +# If SqlAlchemy should pool database connections. +{{ if not .core.sql_alchemy_pool_enabled }}#{{ end }}sql_alchemy_pool_enabled = {{ .core.sql_alchemy_pool_enabled | default "" }} + # The SqlAlchemy pool size is the maximum number of database connections # in the pool. {{ if not .core.sql_alchemy_pool_size }}#{{ end }}sql_alchemy_pool_size = {{ .core.sql_alchemy_pool_size | default "" }} @@ -69,6 +92,10 @@ # not apply to sqlite. {{ if not .core.sql_alchemy_pool_recycle }}#{{ end }}sql_alchemy_pool_recycle = {{ .core.sql_alchemy_pool_recycle | default "" }} +# How many seconds to retry re-establishing a DB connection after +# disconnects. Setting this to 0 disables retries. +{{ if not .core.sql_alchemy_reconnect_timeout }}#{{ end }}sql_alchemy_reconnect_timeout = {{ .core.sql_alchemy_reconnect_timeout | default "" }} + # The amount of parallelism as a setting to the executor. This defines # the max number of task instances that should run simultaneously # on this airflow installation @@ -77,10 +104,10 @@ # The number of task instances allowed to run concurrently by the scheduler {{ if not .core.dag_concurrency }}#{{ end }}dag_concurrency = {{ .core.dag_concurrency | default "" }} -# Are DAGs paused by at creation +# Are DAGs paused by default at creation {{ if not .core.dags_are_paused_at_creation }}#{{ end }}dags_are_paused_at_creation = {{ .core.dags_are_paused_at_creation | default "" }} -# When not using pools, tasks are run in the pool", +# When not using pools, tasks are run in the "default pool", # whose size is guided by this config element {{ if not .core.non_pooled_task_slot_count }}#{{ end }}non_pooled_task_slot_count = {{ .core.non_pooled_task_slot_count | default "" }} @@ -114,15 +141,35 @@ # What security module to use (for example kerberos): {{ if not .core.security }}#{{ end }}security = {{ .core.security | default "" }} +# If set to False enables some unsecure features like Charts and Ad Hoc Queries. +# In 2.0 will default to True. +{{ if not .core.secure_mode }}#{{ end }}secure_mode = {{ .core.secure_mode | default "" }} + # Turn unit test mode on (overwrites many configuration options with test # values at runtime) {{ if not .core.unit_test_mode }}#{{ end }}unit_test_mode = {{ .core.unit_test_mode | default "" }} +# Name of handler to read task instance logs. +# Default to use task handler. +{{ if not .core.task_log_reader }}#{{ end }}task_log_reader = {{ .core.task_log_reader | default "" }} + +# Whether to enable pickling for xcom (note that this is insecure and allows for +# RCE exploits). This will be deprecated in Airflow 2.0 (be forced to False). +{{ if not .core.enable_xcom_pickling }}#{{ end }}enable_xcom_pickling = {{ .core.enable_xcom_pickling | default "" }} + +# When a task is killed forcefully, this is the amount of time in seconds that +# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED +{{ if not .core.killed_task_cleanup_time }}#{{ end }}killed_task_cleanup_time = {{ .core.killed_task_cleanup_time | default "" }} + [cli] # In what way should the cli access the API. The LocalClient will use the # database directly, while the json_client will use the api running on the # webserver {{ if not .cli.api_client }}#{{ end }}api_client = {{ .cli.api_client | default "" }} + +# If you set web_server_url_prefix, do NOT forget to append it here, ex: +# endpoint_url = http://localhost:8080/myroot +# So api will look like: http://localhost:8080/myroot/api/experimental/... {{ if not .cli.endpoint_url }}#{{ end }}endpoint_url = {{ .cli.endpoint_url | default "" }} [api] @@ -138,6 +185,10 @@ {{ if not .operators.default_disk }}#{{ end }}default_disk = {{ .operators.default_disk | default "" }} {{ if not .operators.default_gpus }}#{{ end }}default_gpus = {{ .operators.default_gpus | default "" }} +[hive] +# Default mapreduce queue for HiveOperator tasks +# default_hive_mapred_queue = + [webserver] # The base url of your website as airflow cannot guess what domain or # cname you are using. This is use in automated emails that @@ -155,7 +206,7 @@ {{ if not .webserver.web_server_ssl_cert }}#{{ end }}web_server_ssl_cert = {{ .webserver.web_server_ssl_cert | default "" }} {{ if not .webserver.web_server_ssl_key }}#{{ end }}web_server_ssl_key = {{ .webserver.web_server_ssl_key | default "" }} -# The time the gunicorn webserver waits before timing out on a worker +# Number of seconds the gunicorn webserver waits before timing out on a worker {{ if not .webserver.web_server_worker_timeout }}#{{ end }}web_server_worker_timeout = {{ .webserver.web_server_worker_timeout | default "" }} # Number of workers to refresh at a time. When set to 0, worker refresh is @@ -173,7 +224,7 @@ {{ if not .webserver.workers }}#{{ end }}workers = {{ .webserver.workers | default "" }} # The worker class gunicorn should use. Choices include -# sync ), eventlet, gevent +# sync (default), eventlet, gevent {{ if not .webserver.worker_class }}#{{ end }}worker_class = {{ .webserver.worker_class | default "" }} # Log files for the gunicorn webserver. '-' means log to stderr. @@ -196,6 +247,10 @@ # in order to user the ldapgroup mode. {{ if not .webserver.owner_mode }}#{{ end }}owner_mode = {{ .webserver.owner_mode | default "" }} +# Default DAG view. Valid values are: +# tree, graph, duration, gantt, landing_times +{{ if not .webserver.dag_default_view }}#{{ end }}dag_default_view = {{ .webserver.dag_default_view | default "" }} + # Default DAG orientation. Valid values are: # LR (Left->Right), TB (Top->Bottom), RL (Right->Left), BT (Bottom->Top) {{ if not .webserver.dag_orientation }}#{{ end }}dag_orientation = {{ .webserver.dag_orientation | default "" }} @@ -208,10 +263,13 @@ # while fetching logs from other worker machine {{ if not .webserver.log_fetch_timeout_sec }}#{{ end }}log_fetch_timeout_sec = {{ .webserver.log_fetch_timeout_sec | default "" }} -# By, the webserver shows paused DAGs. Flip this to hide paused -# DAGs by +# By default, the webserver shows paused DAGs. Flip this to hide paused +# DAGs by default {{ if not .webserver.hide_paused_dags_by_default }}#{{ end }}hide_paused_dags_by_default = {{ .webserver.hide_paused_dags_by_default | default "" }} +# Consistent page size across all listing views in the UI +{{ if not .webserver.page_size }}#{{ end }}page_size = {{ .webserver.page_size | default "" }} + [email] {{ if not .email.email_backend }}#{{ end }}email_backend = {{ .email.email_backend | default "" }} @@ -221,10 +279,11 @@ # server here {{ if not .smtp.smtp_host }}#{{ end }}smtp_host = {{ .smtp.smtp_host | default "" }} {{ if not .smtp.smtp_starttls }}#{{ end }}smtp_smtp_starttls = {{ .smtp.smtp_starttls | default "" }} -smtp_ssl = {{ .smtp.smtp_ssl | default "" }} -{{ if not .smtp.smtp_user }}#{{ end }}smtp_user = {{ .smtp.smtp_user | default "" }} +{{ if not .smtp.smtp_ssl }}#{{ end }}smtp_ssl = {{ .smtp.smtp_ssl | default "" }} +# Uncomment and set the user/pass settings if you want to use SMTP AUTH +# {{ if not .smtp.smtp_user }}#{{ end }}smtp_user = {{ .smtp.smtp_user | default "" }} +# {{ if not .smtp.smtp_password }}#{{ end }}smtp_password = {{ .smtp.smtp_password | default "" }} {{ if not .smtp.smtp_port }}#{{ end }}smtp_port = {{ .smtp.smtp_port | default "" }} -{{ if not .smtp.smtp_password }}#{{ end }}smtp_password = {{ .smtp.smtp_password | default "" }} {{ if not .smtp.smtp_mail_from }}#{{ end }}smtp_mail_from = {{ .smtp.smtp_mail_from | default "" }} [celery] @@ -250,21 +309,54 @@ smtp_ssl = {{ .smtp.smtp_ssl | default "" }} # The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally # a sqlalchemy database. Refer to the Celery documentation for more # information. +# http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings {{ if not .celery.broker_url }}#{{ end }}broker_url = {{ .celery.broker_url | default "" }} -# Another key Celery setting -{{ if not .celery.celery_result_backend }}#{{ end }}celery_result_backend = {{ .celery.celery_result_backend | default "" }} +# The Celery result_backend. When a job finishes, it needs to update the +# metadata of the job. Therefore it will post a message on a message bus, +# or insert it into a database (depending of the backend) +# This status is used by the scheduler to update the state of the task +# The use of a database is highly recommended +# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings +{{ if not .celery.result_backend }}#{{ end }}result_backend = {{ .celery.result_backend | default "" }} # Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start # it `airflow flower`. This defines the IP that Celery Flower runs on {{ if not .celery.flower_host }}#{{ end }}flower_host = {{ .celery.flower_host | default "" }} +# The root URL for Flower +# Ex: flower_url_prefix = /flower +{{ if not .celery.flower_url_prefix }}#{{ end }}flower_url_prefix = {{ .celery.flower_url_prefix | default "" }} + # This defines the port that Celery Flower runs on {{ if not .celery.flower_port }}#{{ end }}flower_port = {{ .celery.flower_port | default "" }} # Default queue that tasks get assigned to and that worker listen on. {{ if not .celery.default_queue }}#{{ end }}default_queue = {{ .celery.default_queue | default "" }} +# Import path for celery configuration options +{{ if not .celery.celery_config_options }}#{{ end }}celery_config_options = {{ .celery.celery_config_options | default "" }} + +[celery_broker_transport_options] +# The visibility timeout defines the number of seconds to wait for the worker +# to acknowledge the task before the message is redelivered to another worker. +# Make sure to increase the visibility timeout to match the time of the longest +# ETA you're planning to use. Especially important in case of using Redis or SQS +{{ if not .celery_broker_transport_options.visibility_timeout }}#{{ end }}visibility_timeout = {{ .celery_broker_transport_options.visibility_timeout | default "" }} + +# In case of using SSL +{{ if not .celery_broker_transport_options.ssl_active }}#{{ end }}ssl_active = {{ .celery_broker_transport_options.ssl_active | default "" }} +{{ if not .celery_broker_transport_options.ssl_key }}#{{ end }}ssl_key = {{ .celery_broker_transport_options.ssl_key | default "" }} +{{ if not .celery_broker_transport_options.ssl_cert }}#{{ end }}ssl_cert = {{ .celery_broker_transport_options.ssl_cert | default "" }} +{{ if not .celery_broker_transport_options.ssl_cacert }}#{{ end }}ssl_cacert = {{ .celery_broker_transport_options.ssl_cacert | default "" }} + +[dask] +# This section only applies if you are using the DaskExecutor in +# [core] section above + +# The IP address and port of the Dask cluster's scheduler. +# cluster_address = 127.0.0.1:8786 + [scheduler] # Task instances listen for external kill signal (when you clear tasks # from the CLI or the UI), this defines the frequency at which they should @@ -303,6 +395,11 @@ smtp_ssl = {{ .smtp.smtp_ssl | default "" }} # DAG definition (catchup) {{ if not .scheduler.catchup_by_default }}#{{ end }}catchup_by_default = {{ .scheduler.catchup_by_default | default "" }} +# This changes the batch size of queries in the scheduling main loop. +# This depends on query length limits and how long you are willing to hold locks. +# 0 for no limit +{{ if not .scheduler.max_tis_per_query }}#{{ end }}max_tis_per_query = {{ .scheduler.max_tis_per_query | default "" }} + # Statsd (https://github.com/etsy/statsd) integration settings # statsd_on = False # statsd_host = localhost @@ -310,32 +407,45 @@ smtp_ssl = {{ .smtp.smtp_ssl | default "" }} # statsd_prefix = airflow # The scheduler can run multiple threads in parallel to schedule dags. -# This defines how many threads will run. However airflow will never -# use more threads than the amount of cpu cores available. +# This defines how many threads will run. {{ if not .scheduler.max_threads }}#{{ end }}max_threads = {{ .scheduler.max_threads | default "" }} {{ if not .scheduler.authenticate }}#{{ end }}authenticate = {{ .scheduler.authenticate | default "" }} +[ldap] +# set this to ldaps://: +{{ if not .ldap.uri }}#{{ end }}uri = {{ .ldap.uri | default "" }} +{{ if not .ldap.user_filter }}#{{ end }}user_filter = {{ .ldap.user_filter | default "" }} +{{ if not .ldap.user_name_attr }}#{{ end }}user_name_attr = {{ .ldap.user_name_attr | default "" }} +{{ if not .ldap.group_member_attr }}#{{ end }}group_member_attr = {{ .ldap.group_member_attr | default "" }} +{{ if not .ldap.superuser_filter }}#{{ end }}superuser_filter = {{ .ldap.superuser_filter | default "" }} +{{ if not .ldap.data_profiler_filter }}#{{ end }}data_profiler_filter = {{ .ldap.data_profiler_filter | default "" }} +{{ if not .ldap.bind_user }}#{{ end }}bind_user = {{ .ldap.bind_user | default "" }} +{{ if not .ldap.bind_password }}#{{ end }}bind_password = {{ .ldap.bind_password | default "" }} +{{ if not .ldap.basedn }}#{{ end }}basedn = {{ .ldap.basedn | default "" }} +{{ if not .ldap.cacert }}#{{ end }}cacert = {{ .ldap.cacert | default "" }} +{{ if not .ldap.search_scope }}#{{ end }}search_scope = {{ .ldap.search_scope | default "" }} + [mesos] # Mesos master address which MesosExecutor will connect to. -{{ if not .mesos.master }}#{{ end }}master = {{ .mesos.master | default "" }} +# master = localhost:5050 # The framework name which Airflow scheduler will register itself as on mesos -{{ if not .mesos.framework_name }}#{{ end }}framework_name = {{ .mesos.framework_name | default "" }} +# framework_name = Airflow # Number of cpu cores required for running one task instance using # 'airflow run --local -p ' # command on a mesos slave -{{ if not .mesos.task_cpu }}#{{ end }}task_cpu = {{ .mesos.task_cpu | default "" }} +# task_cpu = 1 # Memory in MB required for running one task instance using # 'airflow run --local -p ' # command on a mesos slave -{{ if not .mesos.task_memory }}#{{ end }}task_memory = {{ .mesos.task_memory | default "" }} +# task_memory = 256 # Enable framework checkpointing for mesos # See http://mesos.apache.org/documentation/latest/slave-recovery/ -{{ if not .mesos.checkpoint }}#{{ end }}checkpoint = {{ .mesos.checkpoint | default "" }} +# checkpoint = False # Failover timeout in milliseconds. # When checkpointing is enabled and this option is set, Mesos waits @@ -347,22 +457,22 @@ smtp_ssl = {{ .smtp.smtp_ssl | default "" }} # Enable framework authentication for mesos # See http://mesos.apache.org/documentation/latest/configuration/ -{{ if not .mesos.authenticate }}#{{ end }}authenticate = {{ .mesos.authenticate | default "" }} +# authenticate = False # Mesos credentials, if authentication is enabled -#_principal = admin -#_secret = admin +# default_principal = admin +# default_secret = admin [kerberos] -#ccache = /tmp/airflow_krb5_ccache +# ccache = /tmp/airflow_krb5_ccache # gets augmented with fqdn -#principal = airflow -#reinit_frequency = 3600 -#kinit_path = kinit -#keytab = airflow.keytab +# principal = airflow +# reinit_frequency = 3600 +# kinit_path = kinit +# keytab = airflow.keytab [github_enterprise] -#api_rev = v3 +# api_rev = v3 [admin] # UI to hide sensitive variable fields when set to True diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index c684521b..519b8031 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -364,15 +364,22 @@ conf: airflow_home: /usr/local/airflow dags_folder: /usr/local/airflow/dags base_log_folder: /usr/local/airflow/logs - remote_base_log_folder: + remote_logging: "False" remote_log_conn_id: encrypt_s3_logs: "False" - executor: CeleryExecutor + logging_level: "INFO" + logging_config_class: + log_format: "[%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s" + simple_log_format: "%%(asctime)s %%(levelname)s - %%(message)s" + default_timezone: "utc" + executor: "CeleryExecutor" + sql_alchemy_pool_enabled: "True" sql_alchemy_pool_size: 5 sql_alchemy_pool_recycle: 3600 + sql_alchemy_reconnect_timeout: 30 parallelism: 32 dag_concurrency: 16 - dags_are_paused_at_creation: "False" + dags_are_paused_at_creation: "True" non_pooled_task_slot_count: 128 max_active_runs_per_dag: 16 load_examples: "False" @@ -380,16 +387,20 @@ conf: fernet_key: fKp7omMJ4QlTxfZzVBSiyXVgeCK-6epRjGgMpEIsjvs= donot_pickle: "False" dagbag_import_timeout: 30 - task_runner: BashTaskRunner + task_runner: "BashTaskRunner" default_impersonation: security: + secure_mode: "True" unit_test_mode: "False" + task_log_reader: "task" + enable_xcom_pickling: "False" + killed_task_cleanup_time: 60 cli: api_client: airflow.api.client.local_client api: auth_backend: airflow.api.auth.backend.default operators: - default_owner: Airflow + default_owner: "Airflow" default_cpus: 1 default_ram: 512 default_disk: 512 @@ -401,36 +412,47 @@ conf: web_server_ssl_key: web_server_worker_timeout: 120 worker_refresh_batch_size: 1 - secret_key: temporary_key + worker_refresh_interval: 30 + secret_key: "temporary_key" workers: 4 - worker_class: sync + worker_class: "sync" access_logfile: "-" error_logfile: "-" - expose_config: "True" + expose_config: "False" authenticate: "False" filter_by_owner: "False" - owner_mode: user - dag_orientation: LR + owner_mode: "user" + dag_default_view: "tree" + dag_orientation: "LR" demo_mode: "False" log_fetch_timeout_sec: 5 hide_paused_dags_by_default: "False" + page_size: 100 email: email_backend: airflow.utils.send_email_smtp smtp: - smtp_host: localhost + smtp_host: "localhost" smtp_starttls: "True" smtp_ssl: "False" - smtp_user: airflow + smtp_user: "airflow" smtp_port: 25 - smtp_password: airflow + smtp_password: "airflow" smtp_mail_from: airflow@airflow.local celery: celery_app_name: airflow.executors.celery_executor celeryd_concurrency: 16 worker_log_server_port: 8793 flower_host: 0.0.0.0 + flower_url_prefix: flower_port: 5555 - default_queue: default + default_queue: "default" + celery_config_options: airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG + celery_broker_transport_options: + visibility_timeout: 21600 + ssl_active: "False" + ssl_key: + ssl_cert: + ssl_cacert: scheduler: job_heartbeat_sec: 5 scheduler_heartbeat_sec: 5 @@ -441,15 +463,21 @@ conf: child_process_log_directory: /usr/local/airflow/logs/scheduler scheduler_zombie_task_threshold: 300 catchup_by_default: "True" + max_tis_per_query: 0 max_threads: 2 authenticate: "False" - mesos: - master: localhost:5050 - framework_name: Airflow - task_cpu: 1 - task_memory: 256 - checkpoint: "False" - authenticate: "False" + ldap: + uri: + user_filter: + user_name_attr: + group_member_attr: + superuser_filter: + data_profiler_filter: + bind_user: + bind_password: + basedn: + cacert: + search_scope: admin: hide_sensitive_variable_fields: "True"