Update Airflow Config Template
There has been significant changes in the Airflow code base with recent software updates. This has resulted in huge changes in airflow.cfg This patch set is meant to align the config file with that of Airflow 1.9.0 [0] [0] https://github.com/apache/incubator-airflow/blob/master/airflow/config_templates/default_airflow.cfg Change-Id: I796fb1803c0f80a7486155864fe0a2a87e7a5737
This commit is contained in:
Anthony Lin
parent
f6d05cc966
commit
7ffc8637fc
|
|
@ -18,25 +18,25 @@
|
|||
|
||||
# Add endpoint URI lookup for Airflow Web Server
|
||||
{{- if empty .Values.conf.airflow.webserver.base_url -}}
|
||||
{{- tuple "airflow_web" "internal" "airflow_web" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup"| set .Values.conf.airflow.webserver "base_url" | quote | trunc 0 -}}
|
||||
{{- tuple "airflow_web" "internal" "airflow_web" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.airflow.webserver "base_url" | quote | trunc 0 -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- if empty .Values.conf.airflow.cli.endpoint_url -}}
|
||||
{{- tuple "airflow_web" "internal" "airflow_web" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup"| set .Values.conf.airflow.cli "endpoint_url" | quote | trunc 0 -}}
|
||||
{{- tuple "airflow_web" "internal" "airflow_web" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.airflow.cli "endpoint_url" | quote | trunc 0 -}}
|
||||
{{- end -}}
|
||||
|
||||
# Add endpoint URI lookup for Postgresql DB Connection
|
||||
{{- if empty .Values.conf.airflow.core.sql_alchemy_conn -}}
|
||||
{{- tuple "postgresql_airflow_db" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup"| set .Values.conf.airflow.core "sql_alchemy_conn" | quote | trunc 0 -}}
|
||||
{{- tuple "postgresql_airflow_db" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.airflow.core "sql_alchemy_conn" | quote | trunc 0 -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- if empty .Values.conf.airflow.celery.result_backend -}}
|
||||
{{- tuple "postgresql_airflow_db" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.airflow.celery "result_backend" | quote | trunc 0 -}}
|
||||
{{- end -}}
|
||||
|
||||
# Add endpoint URI lookup for RabbitMQ Connection
|
||||
{{- if empty .Values.conf.airflow.celery.broker_url -}}
|
||||
{{- tuple "oslo_messaging" "internal" "user" "amqp" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup"| set .Values.conf.airflow.celery "broker_url" | quote | trunc 0 -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- if empty .Values.conf.airflow.celery.celery_result_backend -}}
|
||||
{{- tuple "oslo_messaging" "internal" "user" "amqp" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup"| set .Values.conf.airflow.celery "celery_result_backend" | quote | trunc 0 -}}
|
||||
{{- tuple "oslo_messaging" "internal" "user" "amqp" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.airflow.celery "broker_url" | quote | trunc 0 -}}
|
||||
{{- end -}}
|
||||
|
||||
---
|
||||
|
|
|
|||
|
|
@ -18,41 +18,61 @@
|
|||
{{- define "airflow.conf.airflow_values_skeleton" -}}
|
||||
|
||||
{{- if not .core -}}{{- set . "core" dict -}}{{- end -}}
|
||||
{{- if not .cli -}}{{- set . "cli" dict -}}{{- end -}}
|
||||
{{- if not .api -}}{{- set . "api" dict -}}{{- end -}}
|
||||
{{- if not .operators -}}{{- set . "operators" dict -}}{{- end -}}
|
||||
{{- if not .webserver -}}{{- set . "webserver" dict -}}{{- end -}}
|
||||
{{- if not .email -}}{{- set . "email" dict -}}{{- end -}}
|
||||
{{- if not .smtp -}}{{- set . "smtp" dict -}}{{- end -}}
|
||||
{{- if not .celery -}}{{- set . "celery" dict -}}{{- end -}}
|
||||
{{- if not .celery_broker_transport_options -}}{{- set . "celery_broker_transport_options" dict -}}{{- end -}}
|
||||
{{- if not .scheduler -}}{{- set . "scheduler" dict -}}{{- end -}}
|
||||
{{- if not .mesos -}}{{- set . "mesos" dict -}}{{- end -}}
|
||||
{{- if not .ldap -}}{{- set . "ldap" dict -}}{{- end -}}
|
||||
{{- if not .admin -}}{{- set . "admin" dict -}}{{- end -}}
|
||||
|
||||
{{- end -}}
|
||||
|
||||
{{- define "airflow.conf.airflow" -}}
|
||||
|
||||
[core]
|
||||
# The home folder for airflow, is ~/airflow
|
||||
# The home folder for airflow, default is ~/airflow
|
||||
{{ if not .core.airflow_home }}#{{ end }}airflow_home = {{ .core.airflow_home | default "<None>" }}
|
||||
|
||||
# The folder where your airflow pipelines live, most likely a
|
||||
# subfolder in a code repository
|
||||
# This path must be absolute
|
||||
{{ if not .core.dags_folder }}#{{ end }}dags_folder = {{ .core.dags_folder | default "<None>" }}
|
||||
|
||||
# The folder where airflow should store its log files. This location
|
||||
# The folder where airflow should store its log files
|
||||
# This path must be absolute
|
||||
{{ if not .core.base_log_folder }}#{{ end }}base_log_folder = {{ .core.base_log_folder | default "<None>" }}
|
||||
|
||||
# Airflow can store logs remotely in AWS S3 or Google Cloud Storage. Users
|
||||
# must supply a remote location URL (starting with either 's3://...' or
|
||||
# 'gs://...') and an Airflow connection id that provides access to the storage
|
||||
# location.
|
||||
{{ if not .core.remote_base_log_folder }}#{{ end }}remote_base_log_folder = {{ .core.remote_base_log_folder | default "<None>" }}
|
||||
# must supply an Airflow connection id that provides access to the storage
|
||||
# location. If remote_logging is set to true, see UPDATING.md for additional
|
||||
# configuration requirements.
|
||||
{{ if not .core.remote_logging }}#{{ end }}remote_logging = {{ .core.remote_logging | default "<None>" }}
|
||||
{{ if not .core.remote_log_conn_id }}#{{ end }}remote_log_conn_id = {{ .core.remote_log_conn_id | default "<None>" }}
|
||||
# Use server-side encryption for logs stored in S3
|
||||
{{ if not .core.encrypt_s3_logs }}#{{ end }}encrypt_s3_logs = {{ .core.encrypt_s3_logs | default "<None>" }}
|
||||
# deprecated option for remote log storage, use remote_base_log_folder instead!
|
||||
# s3_log_folder =
|
||||
|
||||
# Logging level
|
||||
{{ if not .core.logging_level }}#{{ end }}logging_level = {{ .core.logging_level | default "<None>" }}
|
||||
|
||||
# Logging class
|
||||
# Specify the class that will specify the logging configuration
|
||||
# This class has to be on the python classpath
|
||||
{{ if not .core.logging_config_class }}#{{ end }}logging_config_class = {{ .core.logging_config_class | default "<None>" }}
|
||||
|
||||
# Log format
|
||||
{{ if not .core.log_format }}#{{ end }}log_format = {{ .core.log_format | default "<None>" }}
|
||||
{{ if not .core.simple_log_format }}#{{ end }}simple_log_format = {{ .core.simple_log_format | default "<None>" }}
|
||||
|
||||
# Default timezone in case supplied date times are naive
|
||||
# can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam)
|
||||
{{ if not .core.default_timezone }}#{{ end }}default_timezone = {{ .core.default_timezone | default "<None>" }}
|
||||
|
||||
# The executor class that airflow should use. Choices include
|
||||
# SequentialExecutor, LocalExecutor, CeleryExecutor
|
||||
# SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor
|
||||
{{ if not .core.executor }}#{{ end }}executor = {{ .core.executor | default "<None>" }}
|
||||
|
||||
# The SqlAlchemy connection string to the metadata database.
|
||||
|
|
@ -60,6 +80,9 @@
|
|||
# their website
|
||||
{{ if not .core.sql_alchemy_conn }}#{{ end }}sql_alchemy_conn = {{ .core.sql_alchemy_conn | default "<None>" }}
|
||||
|
||||
# If SqlAlchemy should pool database connections.
|
||||
{{ if not .core.sql_alchemy_pool_enabled }}#{{ end }}sql_alchemy_pool_enabled = {{ .core.sql_alchemy_pool_enabled | default "<None>" }}
|
||||
|
||||
# The SqlAlchemy pool size is the maximum number of database connections
|
||||
# in the pool.
|
||||
{{ if not .core.sql_alchemy_pool_size }}#{{ end }}sql_alchemy_pool_size = {{ .core.sql_alchemy_pool_size | default "<None>" }}
|
||||
|
|
@ -69,6 +92,10 @@
|
|||
# not apply to sqlite.
|
||||
{{ if not .core.sql_alchemy_pool_recycle }}#{{ end }}sql_alchemy_pool_recycle = {{ .core.sql_alchemy_pool_recycle | default "<None>" }}
|
||||
|
||||
# How many seconds to retry re-establishing a DB connection after
|
||||
# disconnects. Setting this to 0 disables retries.
|
||||
{{ if not .core.sql_alchemy_reconnect_timeout }}#{{ end }}sql_alchemy_reconnect_timeout = {{ .core.sql_alchemy_reconnect_timeout | default "<None>" }}
|
||||
|
||||
# The amount of parallelism as a setting to the executor. This defines
|
||||
# the max number of task instances that should run simultaneously
|
||||
# on this airflow installation
|
||||
|
|
@ -77,10 +104,10 @@
|
|||
# The number of task instances allowed to run concurrently by the scheduler
|
||||
{{ if not .core.dag_concurrency }}#{{ end }}dag_concurrency = {{ .core.dag_concurrency | default "<None>" }}
|
||||
|
||||
# Are DAGs paused by at creation
|
||||
# Are DAGs paused by default at creation
|
||||
{{ if not .core.dags_are_paused_at_creation }}#{{ end }}dags_are_paused_at_creation = {{ .core.dags_are_paused_at_creation | default "<None>" }}
|
||||
|
||||
# When not using pools, tasks are run in the pool",
|
||||
# When not using pools, tasks are run in the "default pool",
|
||||
# whose size is guided by this config element
|
||||
{{ if not .core.non_pooled_task_slot_count }}#{{ end }}non_pooled_task_slot_count = {{ .core.non_pooled_task_slot_count | default "<None>" }}
|
||||
|
||||
|
|
@ -114,15 +141,35 @@
|
|||
# What security module to use (for example kerberos):
|
||||
{{ if not .core.security }}#{{ end }}security = {{ .core.security | default "<None>" }}
|
||||
|
||||
# If set to False enables some unsecure features like Charts and Ad Hoc Queries.
|
||||
# In 2.0 will default to True.
|
||||
{{ if not .core.secure_mode }}#{{ end }}secure_mode = {{ .core.secure_mode | default "<None>" }}
|
||||
|
||||
# Turn unit test mode on (overwrites many configuration options with test
|
||||
# values at runtime)
|
||||
{{ if not .core.unit_test_mode }}#{{ end }}unit_test_mode = {{ .core.unit_test_mode | default "<None>" }}
|
||||
|
||||
# Name of handler to read task instance logs.
|
||||
# Default to use task handler.
|
||||
{{ if not .core.task_log_reader }}#{{ end }}task_log_reader = {{ .core.task_log_reader | default "<None>" }}
|
||||
|
||||
# Whether to enable pickling for xcom (note that this is insecure and allows for
|
||||
# RCE exploits). This will be deprecated in Airflow 2.0 (be forced to False).
|
||||
{{ if not .core.enable_xcom_pickling }}#{{ end }}enable_xcom_pickling = {{ .core.enable_xcom_pickling | default "<None>" }}
|
||||
|
||||
# When a task is killed forcefully, this is the amount of time in seconds that
|
||||
# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED
|
||||
{{ if not .core.killed_task_cleanup_time }}#{{ end }}killed_task_cleanup_time = {{ .core.killed_task_cleanup_time | default "<None>" }}
|
||||
|
||||
[cli]
|
||||
# In what way should the cli access the API. The LocalClient will use the
|
||||
# database directly, while the json_client will use the api running on the
|
||||
# webserver
|
||||
{{ if not .cli.api_client }}#{{ end }}api_client = {{ .cli.api_client | default "<None>" }}
|
||||
|
||||
# If you set web_server_url_prefix, do NOT forget to append it here, ex:
|
||||
# endpoint_url = http://localhost:8080/myroot
|
||||
# So api will look like: http://localhost:8080/myroot/api/experimental/...
|
||||
{{ if not .cli.endpoint_url }}#{{ end }}endpoint_url = {{ .cli.endpoint_url | default "<None>" }}
|
||||
|
||||
[api]
|
||||
|
|
@ -138,6 +185,10 @@
|
|||
{{ if not .operators.default_disk }}#{{ end }}default_disk = {{ .operators.default_disk | default "<None>" }}
|
||||
{{ if not .operators.default_gpus }}#{{ end }}default_gpus = {{ .operators.default_gpus | default "<None>" }}
|
||||
|
||||
[hive]
|
||||
# Default mapreduce queue for HiveOperator tasks
|
||||
# default_hive_mapred_queue =
|
||||
|
||||
[webserver]
|
||||
# The base url of your website as airflow cannot guess what domain or
|
||||
# cname you are using. This is use in automated emails that
|
||||
|
|
@ -155,7 +206,7 @@
|
|||
{{ if not .webserver.web_server_ssl_cert }}#{{ end }}web_server_ssl_cert = {{ .webserver.web_server_ssl_cert | default "<None>" }}
|
||||
{{ if not .webserver.web_server_ssl_key }}#{{ end }}web_server_ssl_key = {{ .webserver.web_server_ssl_key | default "<None>" }}
|
||||
|
||||
# The time the gunicorn webserver waits before timing out on a worker
|
||||
# Number of seconds the gunicorn webserver waits before timing out on a worker
|
||||
{{ if not .webserver.web_server_worker_timeout }}#{{ end }}web_server_worker_timeout = {{ .webserver.web_server_worker_timeout | default "<None>" }}
|
||||
|
||||
# Number of workers to refresh at a time. When set to 0, worker refresh is
|
||||
|
|
@ -173,7 +224,7 @@
|
|||
{{ if not .webserver.workers }}#{{ end }}workers = {{ .webserver.workers | default "<None>" }}
|
||||
|
||||
# The worker class gunicorn should use. Choices include
|
||||
# sync ), eventlet, gevent
|
||||
# sync (default), eventlet, gevent
|
||||
{{ if not .webserver.worker_class }}#{{ end }}worker_class = {{ .webserver.worker_class | default "<None>" }}
|
||||
|
||||
# Log files for the gunicorn webserver. '-' means log to stderr.
|
||||
|
|
@ -196,6 +247,10 @@
|
|||
# in order to user the ldapgroup mode.
|
||||
{{ if not .webserver.owner_mode }}#{{ end }}owner_mode = {{ .webserver.owner_mode | default "<None>" }}
|
||||
|
||||
# Default DAG view. Valid values are:
|
||||
# tree, graph, duration, gantt, landing_times
|
||||
{{ if not .webserver.dag_default_view }}#{{ end }}dag_default_view = {{ .webserver.dag_default_view | default "<None>" }}
|
||||
|
||||
# Default DAG orientation. Valid values are:
|
||||
# LR (Left->Right), TB (Top->Bottom), RL (Right->Left), BT (Bottom->Top)
|
||||
{{ if not .webserver.dag_orientation }}#{{ end }}dag_orientation = {{ .webserver.dag_orientation | default "<None>" }}
|
||||
|
|
@ -208,10 +263,13 @@
|
|||
# while fetching logs from other worker machine
|
||||
{{ if not .webserver.log_fetch_timeout_sec }}#{{ end }}log_fetch_timeout_sec = {{ .webserver.log_fetch_timeout_sec | default "<None>" }}
|
||||
|
||||
# By, the webserver shows paused DAGs. Flip this to hide paused
|
||||
# DAGs by
|
||||
# By default, the webserver shows paused DAGs. Flip this to hide paused
|
||||
# DAGs by default
|
||||
{{ if not .webserver.hide_paused_dags_by_default }}#{{ end }}hide_paused_dags_by_default = {{ .webserver.hide_paused_dags_by_default | default "<None>" }}
|
||||
|
||||
# Consistent page size across all listing views in the UI
|
||||
{{ if not .webserver.page_size }}#{{ end }}page_size = {{ .webserver.page_size | default "<None>" }}
|
||||
|
||||
[email]
|
||||
{{ if not .email.email_backend }}#{{ end }}email_backend = {{ .email.email_backend | default "<None>" }}
|
||||
|
||||
|
|
@ -221,10 +279,11 @@
|
|||
# server here
|
||||
{{ if not .smtp.smtp_host }}#{{ end }}smtp_host = {{ .smtp.smtp_host | default "<None>" }}
|
||||
{{ if not .smtp.smtp_starttls }}#{{ end }}smtp_smtp_starttls = {{ .smtp.smtp_starttls | default "<None>" }}
|
||||
smtp_ssl = {{ .smtp.smtp_ssl | default "<None>" }}
|
||||
{{ if not .smtp.smtp_user }}#{{ end }}smtp_user = {{ .smtp.smtp_user | default "<None>" }}
|
||||
{{ if not .smtp.smtp_ssl }}#{{ end }}smtp_ssl = {{ .smtp.smtp_ssl | default "<None>" }}
|
||||
# Uncomment and set the user/pass settings if you want to use SMTP AUTH
|
||||
# {{ if not .smtp.smtp_user }}#{{ end }}smtp_user = {{ .smtp.smtp_user | default "<None>" }}
|
||||
# {{ if not .smtp.smtp_password }}#{{ end }}smtp_password = {{ .smtp.smtp_password | default "<None>" }}
|
||||
{{ if not .smtp.smtp_port }}#{{ end }}smtp_port = {{ .smtp.smtp_port | default "<None>" }}
|
||||
{{ if not .smtp.smtp_password }}#{{ end }}smtp_password = {{ .smtp.smtp_password | default "<None>" }}
|
||||
{{ if not .smtp.smtp_mail_from }}#{{ end }}smtp_mail_from = {{ .smtp.smtp_mail_from | default "<None>" }}
|
||||
|
||||
[celery]
|
||||
|
|
@ -250,21 +309,54 @@ smtp_ssl = {{ .smtp.smtp_ssl | default "<None>" }}
|
|||
# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally
|
||||
# a sqlalchemy database. Refer to the Celery documentation for more
|
||||
# information.
|
||||
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings
|
||||
{{ if not .celery.broker_url }}#{{ end }}broker_url = {{ .celery.broker_url | default "<None>" }}
|
||||
|
||||
# Another key Celery setting
|
||||
{{ if not .celery.celery_result_backend }}#{{ end }}celery_result_backend = {{ .celery.celery_result_backend | default "<None>" }}
|
||||
# The Celery result_backend. When a job finishes, it needs to update the
|
||||
# metadata of the job. Therefore it will post a message on a message bus,
|
||||
# or insert it into a database (depending of the backend)
|
||||
# This status is used by the scheduler to update the state of the task
|
||||
# The use of a database is highly recommended
|
||||
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings
|
||||
{{ if not .celery.result_backend }}#{{ end }}result_backend = {{ .celery.result_backend | default "<None>" }}
|
||||
|
||||
# Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start
|
||||
# it `airflow flower`. This defines the IP that Celery Flower runs on
|
||||
{{ if not .celery.flower_host }}#{{ end }}flower_host = {{ .celery.flower_host | default "<None>" }}
|
||||
|
||||
# The root URL for Flower
|
||||
# Ex: flower_url_prefix = /flower
|
||||
{{ if not .celery.flower_url_prefix }}#{{ end }}flower_url_prefix = {{ .celery.flower_url_prefix | default "<None>" }}
|
||||
|
||||
# This defines the port that Celery Flower runs on
|
||||
{{ if not .celery.flower_port }}#{{ end }}flower_port = {{ .celery.flower_port | default "<None>" }}
|
||||
|
||||
# Default queue that tasks get assigned to and that worker listen on.
|
||||
{{ if not .celery.default_queue }}#{{ end }}default_queue = {{ .celery.default_queue | default "<None>" }}
|
||||
|
||||
# Import path for celery configuration options
|
||||
{{ if not .celery.celery_config_options }}#{{ end }}celery_config_options = {{ .celery.celery_config_options | default "<None>" }}
|
||||
|
||||
[celery_broker_transport_options]
|
||||
# The visibility timeout defines the number of seconds to wait for the worker
|
||||
# to acknowledge the task before the message is redelivered to another worker.
|
||||
# Make sure to increase the visibility timeout to match the time of the longest
|
||||
# ETA you're planning to use. Especially important in case of using Redis or SQS
|
||||
{{ if not .celery_broker_transport_options.visibility_timeout }}#{{ end }}visibility_timeout = {{ .celery_broker_transport_options.visibility_timeout | default "<None>" }}
|
||||
|
||||
# In case of using SSL
|
||||
{{ if not .celery_broker_transport_options.ssl_active }}#{{ end }}ssl_active = {{ .celery_broker_transport_options.ssl_active | default "<None>" }}
|
||||
{{ if not .celery_broker_transport_options.ssl_key }}#{{ end }}ssl_key = {{ .celery_broker_transport_options.ssl_key | default "<None>" }}
|
||||
{{ if not .celery_broker_transport_options.ssl_cert }}#{{ end }}ssl_cert = {{ .celery_broker_transport_options.ssl_cert | default "<None>" }}
|
||||
{{ if not .celery_broker_transport_options.ssl_cacert }}#{{ end }}ssl_cacert = {{ .celery_broker_transport_options.ssl_cacert | default "<None>" }}
|
||||
|
||||
[dask]
|
||||
# This section only applies if you are using the DaskExecutor in
|
||||
# [core] section above
|
||||
|
||||
# The IP address and port of the Dask cluster's scheduler.
|
||||
# cluster_address = 127.0.0.1:8786
|
||||
|
||||
[scheduler]
|
||||
# Task instances listen for external kill signal (when you clear tasks
|
||||
# from the CLI or the UI), this defines the frequency at which they should
|
||||
|
|
@ -303,6 +395,11 @@ smtp_ssl = {{ .smtp.smtp_ssl | default "<None>" }}
|
|||
# DAG definition (catchup)
|
||||
{{ if not .scheduler.catchup_by_default }}#{{ end }}catchup_by_default = {{ .scheduler.catchup_by_default | default "<None>" }}
|
||||
|
||||
# This changes the batch size of queries in the scheduling main loop.
|
||||
# This depends on query length limits and how long you are willing to hold locks.
|
||||
# 0 for no limit
|
||||
{{ if not .scheduler.max_tis_per_query }}#{{ end }}max_tis_per_query = {{ .scheduler.max_tis_per_query | default "<None>" }}
|
||||
|
||||
# Statsd (https://github.com/etsy/statsd) integration settings
|
||||
# statsd_on = False
|
||||
# statsd_host = localhost
|
||||
|
|
@ -310,32 +407,45 @@ smtp_ssl = {{ .smtp.smtp_ssl | default "<None>" }}
|
|||
# statsd_prefix = airflow
|
||||
|
||||
# The scheduler can run multiple threads in parallel to schedule dags.
|
||||
# This defines how many threads will run. However airflow will never
|
||||
# use more threads than the amount of cpu cores available.
|
||||
# This defines how many threads will run.
|
||||
{{ if not .scheduler.max_threads }}#{{ end }}max_threads = {{ .scheduler.max_threads | default "<None>" }}
|
||||
|
||||
{{ if not .scheduler.authenticate }}#{{ end }}authenticate = {{ .scheduler.authenticate | default "<None>" }}
|
||||
|
||||
[ldap]
|
||||
# set this to ldaps://<your.ldap.server>:<port>
|
||||
{{ if not .ldap.uri }}#{{ end }}uri = {{ .ldap.uri | default "<None>" }}
|
||||
{{ if not .ldap.user_filter }}#{{ end }}user_filter = {{ .ldap.user_filter | default "<None>" }}
|
||||
{{ if not .ldap.user_name_attr }}#{{ end }}user_name_attr = {{ .ldap.user_name_attr | default "<None>" }}
|
||||
{{ if not .ldap.group_member_attr }}#{{ end }}group_member_attr = {{ .ldap.group_member_attr | default "<None>" }}
|
||||
{{ if not .ldap.superuser_filter }}#{{ end }}superuser_filter = {{ .ldap.superuser_filter | default "<None>" }}
|
||||
{{ if not .ldap.data_profiler_filter }}#{{ end }}data_profiler_filter = {{ .ldap.data_profiler_filter | default "<None>" }}
|
||||
{{ if not .ldap.bind_user }}#{{ end }}bind_user = {{ .ldap.bind_user | default "<None>" }}
|
||||
{{ if not .ldap.bind_password }}#{{ end }}bind_password = {{ .ldap.bind_password | default "<None>" }}
|
||||
{{ if not .ldap.basedn }}#{{ end }}basedn = {{ .ldap.basedn | default "<None>" }}
|
||||
{{ if not .ldap.cacert }}#{{ end }}cacert = {{ .ldap.cacert | default "<None>" }}
|
||||
{{ if not .ldap.search_scope }}#{{ end }}search_scope = {{ .ldap.search_scope | default "<None>" }}
|
||||
|
||||
[mesos]
|
||||
# Mesos master address which MesosExecutor will connect to.
|
||||
{{ if not .mesos.master }}#{{ end }}master = {{ .mesos.master | default "<None>" }}
|
||||
# master = localhost:5050
|
||||
|
||||
# The framework name which Airflow scheduler will register itself as on mesos
|
||||
{{ if not .mesos.framework_name }}#{{ end }}framework_name = {{ .mesos.framework_name | default "<None>" }}
|
||||
# framework_name = Airflow
|
||||
|
||||
# Number of cpu cores required for running one task instance using
|
||||
# 'airflow run <dag_id> <task_id> <execution_date> --local -p <pickle_id>'
|
||||
# command on a mesos slave
|
||||
{{ if not .mesos.task_cpu }}#{{ end }}task_cpu = {{ .mesos.task_cpu | default "<None>" }}
|
||||
# task_cpu = 1
|
||||
|
||||
# Memory in MB required for running one task instance using
|
||||
# 'airflow run <dag_id> <task_id> <execution_date> --local -p <pickle_id>'
|
||||
# command on a mesos slave
|
||||
{{ if not .mesos.task_memory }}#{{ end }}task_memory = {{ .mesos.task_memory | default "<None>" }}
|
||||
# task_memory = 256
|
||||
|
||||
# Enable framework checkpointing for mesos
|
||||
# See http://mesos.apache.org/documentation/latest/slave-recovery/
|
||||
{{ if not .mesos.checkpoint }}#{{ end }}checkpoint = {{ .mesos.checkpoint | default "<None>" }}
|
||||
# checkpoint = False
|
||||
|
||||
# Failover timeout in milliseconds.
|
||||
# When checkpointing is enabled and this option is set, Mesos waits
|
||||
|
|
@ -347,22 +457,22 @@ smtp_ssl = {{ .smtp.smtp_ssl | default "<None>" }}
|
|||
|
||||
# Enable framework authentication for mesos
|
||||
# See http://mesos.apache.org/documentation/latest/configuration/
|
||||
{{ if not .mesos.authenticate }}#{{ end }}authenticate = {{ .mesos.authenticate | default "<None>" }}
|
||||
# authenticate = False
|
||||
|
||||
# Mesos credentials, if authentication is enabled
|
||||
#_principal = admin
|
||||
#_secret = admin
|
||||
# default_principal = admin
|
||||
# default_secret = admin
|
||||
|
||||
[kerberos]
|
||||
#ccache = /tmp/airflow_krb5_ccache
|
||||
# ccache = /tmp/airflow_krb5_ccache
|
||||
# gets augmented with fqdn
|
||||
#principal = airflow
|
||||
#reinit_frequency = 3600
|
||||
#kinit_path = kinit
|
||||
#keytab = airflow.keytab
|
||||
# principal = airflow
|
||||
# reinit_frequency = 3600
|
||||
# kinit_path = kinit
|
||||
# keytab = airflow.keytab
|
||||
|
||||
[github_enterprise]
|
||||
#api_rev = v3
|
||||
# api_rev = v3
|
||||
|
||||
[admin]
|
||||
# UI to hide sensitive variable fields when set to True
|
||||
|
|
|
|||
|
|
@ -364,15 +364,22 @@ conf:
|
|||
airflow_home: /usr/local/airflow
|
||||
dags_folder: /usr/local/airflow/dags
|
||||
base_log_folder: /usr/local/airflow/logs
|
||||
remote_base_log_folder:
|
||||
remote_logging: "False"
|
||||
remote_log_conn_id:
|
||||
encrypt_s3_logs: "False"
|
||||
executor: CeleryExecutor
|
||||
logging_level: "INFO"
|
||||
logging_config_class:
|
||||
log_format: "[%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s"
|
||||
simple_log_format: "%%(asctime)s %%(levelname)s - %%(message)s"
|
||||
default_timezone: "utc"
|
||||
executor: "CeleryExecutor"
|
||||
sql_alchemy_pool_enabled: "True"
|
||||
sql_alchemy_pool_size: 5
|
||||
sql_alchemy_pool_recycle: 3600
|
||||
sql_alchemy_reconnect_timeout: 30
|
||||
parallelism: 32
|
||||
dag_concurrency: 16
|
||||
dags_are_paused_at_creation: "False"
|
||||
dags_are_paused_at_creation: "True"
|
||||
non_pooled_task_slot_count: 128
|
||||
max_active_runs_per_dag: 16
|
||||
load_examples: "False"
|
||||
|
|
@ -380,16 +387,20 @@ conf:
|
|||
fernet_key: fKp7omMJ4QlTxfZzVBSiyXVgeCK-6epRjGgMpEIsjvs=
|
||||
donot_pickle: "False"
|
||||
dagbag_import_timeout: 30
|
||||
task_runner: BashTaskRunner
|
||||
task_runner: "BashTaskRunner"
|
||||
default_impersonation:
|
||||
security:
|
||||
secure_mode: "True"
|
||||
unit_test_mode: "False"
|
||||
task_log_reader: "task"
|
||||
enable_xcom_pickling: "False"
|
||||
killed_task_cleanup_time: 60
|
||||
cli:
|
||||
api_client: airflow.api.client.local_client
|
||||
api:
|
||||
auth_backend: airflow.api.auth.backend.default
|
||||
operators:
|
||||
default_owner: Airflow
|
||||
default_owner: "Airflow"
|
||||
default_cpus: 1
|
||||
default_ram: 512
|
||||
default_disk: 512
|
||||
|
|
@ -401,36 +412,47 @@ conf:
|
|||
web_server_ssl_key:
|
||||
web_server_worker_timeout: 120
|
||||
worker_refresh_batch_size: 1
|
||||
secret_key: temporary_key
|
||||
worker_refresh_interval: 30
|
||||
secret_key: "temporary_key"
|
||||
workers: 4
|
||||
worker_class: sync
|
||||
worker_class: "sync"
|
||||
access_logfile: "-"
|
||||
error_logfile: "-"
|
||||
expose_config: "True"
|
||||
expose_config: "False"
|
||||
authenticate: "False"
|
||||
filter_by_owner: "False"
|
||||
owner_mode: user
|
||||
dag_orientation: LR
|
||||
owner_mode: "user"
|
||||
dag_default_view: "tree"
|
||||
dag_orientation: "LR"
|
||||
demo_mode: "False"
|
||||
log_fetch_timeout_sec: 5
|
||||
hide_paused_dags_by_default: "False"
|
||||
page_size: 100
|
||||
email:
|
||||
email_backend: airflow.utils.send_email_smtp
|
||||
smtp:
|
||||
smtp_host: localhost
|
||||
smtp_host: "localhost"
|
||||
smtp_starttls: "True"
|
||||
smtp_ssl: "False"
|
||||
smtp_user: airflow
|
||||
smtp_user: "airflow"
|
||||
smtp_port: 25
|
||||
smtp_password: airflow
|
||||
smtp_password: "airflow"
|
||||
smtp_mail_from: airflow@airflow.local
|
||||
celery:
|
||||
celery_app_name: airflow.executors.celery_executor
|
||||
celeryd_concurrency: 16
|
||||
worker_log_server_port: 8793
|
||||
flower_host: 0.0.0.0
|
||||
flower_url_prefix:
|
||||
flower_port: 5555
|
||||
default_queue: default
|
||||
default_queue: "default"
|
||||
celery_config_options: airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
|
||||
celery_broker_transport_options:
|
||||
visibility_timeout: 21600
|
||||
ssl_active: "False"
|
||||
ssl_key:
|
||||
ssl_cert:
|
||||
ssl_cacert:
|
||||
scheduler:
|
||||
job_heartbeat_sec: 5
|
||||
scheduler_heartbeat_sec: 5
|
||||
|
|
@ -441,15 +463,21 @@ conf:
|
|||
child_process_log_directory: /usr/local/airflow/logs/scheduler
|
||||
scheduler_zombie_task_threshold: 300
|
||||
catchup_by_default: "True"
|
||||
max_tis_per_query: 0
|
||||
max_threads: 2
|
||||
authenticate: "False"
|
||||
mesos:
|
||||
master: localhost:5050
|
||||
framework_name: Airflow
|
||||
task_cpu: 1
|
||||
task_memory: 256
|
||||
checkpoint: "False"
|
||||
authenticate: "False"
|
||||
ldap:
|
||||
uri:
|
||||
user_filter:
|
||||
user_name_attr:
|
||||
group_member_attr:
|
||||
superuser_filter:
|
||||
data_profiler_filter:
|
||||
bind_user:
|
||||
bind_password:
|
||||
basedn:
|
||||
cacert:
|
||||
search_scope:
|
||||
admin:
|
||||
hide_sensitive_variable_fields: "True"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue