# Copyright 2017 The Openstack-Helm Authors. # Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This file provides defaults for shipyard and airflow # Indicate whether it is production or development environment prod_environment: true labels: job: node_selector_key: ucp-control-plane node_selector_value: enabled shipyard: node_selector_key: ucp-control-plane node_selector_value: enabled airflow: node_selector_key: ucp-control-plane node_selector_value: enabled images: tags: airflow: quay.io/airshipit/airflow:latest shipyard: quay.io/airshipit/shipyard:latest dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 shipyard_db_init: docker.io/postgres:9.5 shipyard_db_sync: quay.io/airshipit/shipyard:latest airflow_db_init: docker.io/postgres:9.5 rabbit_init: docker.io/rabbitmq:3.7-management airflow_db_sync: quay.io/airshipit/airflow:latest ks_user: docker.io/openstackhelm/heat:ocata ks_service: docker.io/openstackhelm/heat:ocata ks_endpoints: docker.io/openstackhelm/heat:ocata image_repo_sync: docker.io/docker:17.07.0 pull_policy: "IfNotPresent" local_registry: active: false exclude: - dep_check - image_repo_sync release_group: null network: shipyard: ingress: public: true classes: namespace: "nginx" cluster: "nginx-cluster" annotations: nginx.ingress.kubernetes.io/rewrite-target: / nginx.ingress.kubernetes.io/proxy-read-timeout: "600" nginx.ingress.kubernetes.io/proxy-body-size: "10m" nginx.ingress.kubernetes.io/configuration-snippet: | more_clear_headers "Server"; node_port: 31901 enable_node_port: false airflow: ingress: public: true proxy_read_timeout: 600 flower: name: airflow-flower port: 5555 enable_node_port: false web: name: airflow-web port: 8080 node_port: 32080 enable_node_port: false worker: name: airflow-worker port: 8793 enable_node_port: false dependencies: static: shipyard_db_init: jobs: - airflow-db-init - airflow-db-sync services: - service: postgresql_shipyard_db endpoint: internal - service: airflow_flower endpoint: internal - service: airflow_web endpoint: internal shipyard_db_sync: jobs: - shipyard-db-init services: - service: postgresql_shipyard_db endpoint: internal airflow_db_init: services: - service: postgresql_airflow_db endpoint: internal rabbit_init: services: - service: oslo_messaging endpoint: internal airflow_db_sync: jobs: - airflow-db-init services: - service: postgresql_airflow_db endpoint: internal ks_user: services: - service: identity endpoint: internal ks_service: services: - service: identity endpoint: internal ks_endpoints: jobs: - shipyard-ks-service services: - service: identity endpoint: internal shipyard: jobs: - shipyard-db-init - shipyard-db-sync - shipyard-ks-endpoints - shipyard-ks-user - shipyard-ks-endpoints services: - service: airflow_flower endpoint: internal - service: airflow_web endpoint: internal - service: identity endpoint: internal - service: postgresql_shipyard_db endpoint: internal airflow_server: jobs: - airflow-rabbit-init - airflow-db-init - airflow-db-sync services: - service: postgresql_airflow_db endpoint: internal - service: oslo_messaging endpoint: internal volume_worker: class_name: general size: 5Gi logrotate: days_before_deletion: 30 # typically overriden by environmental # values, but should include all endpoints # required by this chart endpoints: cluster_domain_suffix: cluster.local identity: name: keystone auth: shipyard: region_name: RegionOne role: admin project_name: service project_domain_name: default user_domain_name: default username: shipyard password: password admin: region_name: RegionOne project_name: admin password: password username: admin user_domain_name: default project_domain_name: default hosts: default: keystone internal: keystone-api path: default: /v3 scheme: default: http port: api: default: 80 internal: 5000 host_fqdn_override: default: null shipyard: name: shipyard hosts: default: shipyard-int public: shipyard-api port: api: default: 9000 public: 80 path: default: /api/v1.0 scheme: default: http host_fqdn_override: default: null # NOTE(bryan-strassner): this chart supports TLS for fqdn over-ridden public # endpoints using the following format: # public: # host: null # tls: # crt: null # key: null airflow_web: name: airflow-web hosts: default: airflow-web-int public: airflow-web port: airflow_web: default: 8080 public: 80 path: default: / scheme: default: http host_fqdn_override: default: null airflow_flower: name: airflow-flower hosts: default: airflow-flower port: airflow_flower: default: 5555 path: default: / scheme: default: http host_fqdn_override: default: null airflow_worker: name: airflow-worker hosts: default: airflow-worker discovery: airflow-worker-discovery host_fqdn_override: default: null path: null scheme: 'http' port: airflow_worker: default: 8793 postgresql_shipyard_db: name: postgresql_shipyard_db auth: admin: username: postgres password: password user: username: shipyard password: password database: shipyard hosts: default: postgresql path: /shipyard scheme: postgresql+psycopg2 port: postgresql: default: 5432 host_fqdn_override: default: null postgresql_airflow_db: name: postgresql_airflow_db auth: admin: username: postgres password: password user: username: airflow password: password database: airflow hosts: default: postgresql path: /airflow scheme: postgresql+psycopg2 port: postgresql: default: 5432 host_fqdn_override: default: null postgresql_airflow_celery_db: name: postgresql_airflow_celery_db auth: admin: username: postgres password: password user: username: airflow password: password database: airflow hosts: default: postgresql path: /airflow scheme: db+postgresql port: postgresql: default: 5432 host_fqdn_override: default: null oslo_messaging: auth: user: username: airflow password: password admin: username: rabbitmq password: password hosts: default: rabbitmq host_fqdn_override: default: null path: /airflow scheme: amqp port: amqp: default: 5672 http: default: 15672 oslo_cache: hosts: default: memcached host_fqdn_override: default: null port: memcache: default: 11211 secrets: identity: admin: shipyard-keystone-admin shipyard: shipyard-keystone-user oslo_messaging: admin: airflow-rabbitmq-admin airflow: airflow-rabbitmq-user postgresql_shipyard_db: admin: shipyard-db-admin user: shipyard-db-user postgresql_airflow_db: admin: airflow-db-admin user: airflow-db-user tls: shipyard: shipyard: public: shipyard-tls-public conf: uwsgi: threads: 1 workers: 4 policy: admin_create: role:admin or role:admin_ucp admin_read_access: rule:admin_create or role:admin_ucp_viewer workflow_orchestrator:list_actions: rule:admin_read_access workflow_orchestrator:create_action: rule:admin_create workflow_orchestrator:get_action: rule:admin_read_access workflow_orchestrator:get_action_step: rule:admin_read_access workflow_orchestrator:get_action_step_logs: rule:admin_read_access workflow_orchestrator:get_action_validation: rule:admin_read_access workflow_orchestrator:invoke_action_control: rule:admin_create workflow_orchestrator:get_configdocs_status: rule:admin_read_access workflow_orchestrator:create_configdocs: rule:admin_create workflow_orchestrator:get_configdocs: rule:admin_read_access workflow_orchestrator:get_configdocs_cleartext: rule:admin_create workflow_orchestrator:commit_configdocs: rule:admin_create workflow_orchestrator:get_renderedconfigdocs: rule:admin_read_access workflow_orchestrator:get_renderedconfigdocs_cleartext: rule:admin_create workflow_orchestrator:list_workflows: rule:admin_read_access workflow_orchestrator:get_workflow: rule:admin_read_access workflow_orchestrator:get_notedetails: rule:admin_read_access workflow_orchestrator:get_site_statuses: rule:admin_read_access workflow_orchestrator:action_deploy_site: rule:admin_create workflow_orchestrator:action_update_site: rule:admin_create workflow_orchestrator:action_update_software: rule:admin_create workflow_orchestrator:action_redeploy_server: rule:admin_create workflow_orchestrator:action_relabel_nodes: rule:admin_create rabbitmq: # adding rmq policy to mirror messages from celery queues policies: - vhost: "/" name: "ha_celery" definition: ha-mode: "all" ha-sync-mode: "automatic" priority: 0 apply-to: all pattern: 'celery.*' paste: app:shipyard-api: paste.app_factory: shipyard_airflow.shipyard_api:paste_start_shipyard pipeline:main: pipeline: authtoken shipyard-api filter:authtoken: paste.filter_factory: keystonemiddleware.auth_token:filter_factory shipyard: base: web_server: pool_size: 15 pool_pre_ping: true pool_timeout: 30 pool_overflow: 10 connection_recycle: -1 profiler: false shipyard: service_type: shipyard deckhand: service_type: deckhand armada: service_type: armada drydock: service_type: physicalprovisioner promenade: service_type: kubernetesprovisioner keystone_authtoken: delay_auth_decision: true auth_type: password auth_section: keystone_authtoken auth_version: v3 memcache_security_strategy: ENCRYPT requests_config: airflow_log_connect_timeout: 5 airflow_log_read_timeout: 300 deckhand_client_connect_timeout: 5 deckhand_client_read_timeout: 300 validation_connect_timeout: 5 validation_read_timeout: 300 notes_connect_timeout: 5 notes_read_timeout: 10 drydock_client_connect_timeout: 20 drydock_client_read_timeout: 300 airflow: worker_endpoint_scheme: 'http' worker_port: 8793 k8s_logs: ucp_namespace: 'ucp' oslo_policy: policy_file: /etc/shipyard/policy.yaml # If non-existent rule is used, the request should be denied. The # deny_all rule is hard coded in the policy.py code to allow no access. policy_default_rule: deny_all airflow_config_file: path: /usr/local/airflow/airflow.cfg airflow: # NOTE: Airflow 1.10 introduces a need to declare all config options: # https://issues.apache.org/jira/browse/AIRFLOW-3099 core: airflow_home: /usr/local/airflow dags_folder: /usr/local/airflow/dags base_log_folder: /usr/local/airflow/logs remote_logging: "False" remote_log_conn_id: "" remote_base_log_folder: "" encrypt_s3_logs: "False" logging_level: "INFO" fab_logging_level: "WARN" # See image-bundled log_config.py. # Adds console logging of task/step logs. logging_config_class: log_config.LOGGING_CONFIG # NOTE: Airflow 1.10 introduces extra newline characters between log # records. Version 1.10.1 should resolve this issue # https://issues.apache.org/jira/browse/AIRFLOW-1917 # # NOTE: The log format ends up repeated for each log record that we log # in our custom operators, once for the logging_mixin class of # Airflow itself, and once again for the message we want to log. # E.g.: # 2018-09-21 19:38:48,950 INFO logging_mixin(95) write - 2018-09-21 19:38:48,950 INFO deployment_configuration_operator(135) get_doc - Deckhand Client acquired # # NOTE: Updated from default to match Shipyard logging as much as # possible without more aggressive techniques # log_format: "%%(asctime)s %%(levelname)-8s %%(module)s(%%(lineno)d) %%(funcName)s - %%(message)s" simple_log_format: "%%(asctime)s %%(levelname)s - %%(message)s" log_filename_template: "{{ ti.dag_id }}/{{ ti.task_id }}/{{ execution_date.strftime('%%Y-%%m-%%dT%%H:%%M:%%S') }}/{{ try_number }}.log" log_processor_filename_template: "{{ filename }}.log" dag_processor_manager_log_location: /usr/local/airflow/logs/dag_processor_manager/dag_processor_manager.log hostname_callable: "socket:getfqdn" default_timezone: "utc" executor: "CeleryExecutor" # sql_alchemy_conn is extracted from endpoints by the configmap template sql_engine_encoding: "utf-8" sql_alchemy_pool_enabled: "True" sql_alchemy_pool_size: 5 sql_alchemy_pool_recycle: 1800 sql_alchemy_reconnect_timeout: 30 sql_alchemy_schema: "" parallelism: 32 dag_concurrency: 8 dags_are_paused_at_creation: "False" non_pooled_task_slot_count: 128 max_active_runs_per_dag: 8 load_examples: "False" plugins_folder: /usr/local/airflow/plugins fernet_key: fKp7omMJ4QlTxfZzVBSiyXVgeCK-6epRjGgMpEIsjvs= donot_pickle: "False" dagbag_import_timeout: 30 task_runner: "BashTaskRunner" # task_runner: "StandardTaskRunner" -- coming soon? default_impersonation: "" security: "" secure_mode: "True" unit_test_mode: "False" task_log_reader: "task" enable_xcom_pickling: "False" killed_task_cleanup_time: 60 dag_run_conf_overrides_params: "False" worker_precheck: "False" cli: api_client: airflow.api.client.local_client # endpoint_url is extracted from endpoints by the configmap template api: auth_backend: airflow.api.auth.backend.default lineage: # Shipyard is not using this backend: "" atlas: # Shipyard is not using this sasl_enabled: "False" host: "" port: 21000 username: "" password: "" operators: default_owner: "airflow" default_cpus: 1 default_ram: 512 default_disk: 512 default_gpus: 0 hive: # Shipyard is not using this default_hive_mapred_queue: "" webserver: # base_url is extracted from endpoints by the configmap template web_server_host: 0.0.0.0 web_server_port: 8080 web_server_ssl_cert: "" web_server_ssl_key: "" web_server_master_timeout: 120 web_server_worker_timeout: 120 worker_refresh_batch_size: 1 worker_refresh_interval: 120 secret_key: "{SECRET_KEY}" workers: 4 worker_class: "sync" access_logfile: "-" error_logfile: "-" expose_config: "False" authenticate: "False" filter_by_owner: "False" owner_mode: "user" dag_default_view: "tree" dag_orientation: "LR" demo_mode: "False" log_fetch_timeout_sec: 20 hide_paused_dags_by_default: "False" page_size: 100 rbac: "False" navbar_color: "#007A87" default_dag_run_display_number: 25 enable_proxy_fix: "False" email: # Shipyard is not using this email_backend: airflow.utils.send_email_smtp smtp: # Shipyard is not using this smtp_host: "localhost" smtp_starttls: "True" smtp_ssl: "False" smtp_user: "airflow" smtp_password: "airflow" smtp_port: 25 smtp_mail_from: airflow@airflow.local celery: celery_app_name: airflow.executors.celery_executor # The concurrency that will be used when starting workers with the # "airflow worker" command. This defines the number of task instances # that a worker will take, so size up your workers based on the resources # on your worker box and the nature of your tasks worker_concurrency: 4 worker_log_server_port: 8793 # broker_url is extracted from endpoints by the configmap template # result_backend is extracted from endpoints by the configmap template flower_host: 0.0.0.0 flower_url_prefix: "" flower_port: 5555 flower_basic_auth: "" default_queue: "default" # How many processes CeleryExecutor uses to sync task state. # 0 means to use max(1, number of cores - 1) processes. # set to 1 for low-volume use sync_parallelism: 1 celery_config_options: airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG # TODO: Enable this for security ssl_active: "False" ssl_key: "" ssl_cert: "" ssl_cacert: "" celery_broker_transport_options: visibility_timeout: 21600 dask: # Shipyard is not using this cluster_adresss: "127.0.0.1:8786" tls_ca: "" tls_cert: "" tls_key: "" scheduler: # Task instances listen for external kill signal (when you clear tasks # from the CLI or the UI), this defines the frequency at which they # should listen (in seconds). job_heartbeat_sec: 10 # The scheduler constantly tries to trigger new tasks (look at the # scheduler section in the docs for more information). This defines # how often the scheduler should run (in seconds). scheduler_heartbeat_sec: 10 run_duration: -1 # Check for pending dag runs no more than every 10 seconds min_file_process_interval: 10 dag_dir_list_interval: 300 # Stats for the scheduler are minimally useful - every 5 mins is enough print_stats_interval: 300 child_process_log_directory: /usr/local/airflow/logs/scheduler scheduler_zombie_task_threshold: 300 catchup_by_default: "True" max_tis_per_query: 512 statsd_on: "False" statsd_host: "localhost" statsd_port: 8125 statsd_prefix: "airflow" # Shipyard's use of Airflow is low volume. 1 Thread is probably enough. max_threads: 1 authenticate: "False" # Turn off scheduler use of cron intervals by setting this to False. # DAGs submitted manually in the web UI or with trigger_dag will still run. use_job_schedule: "False" ldap: # Shipyard is not using this uri: "" user_filter: "objectClass=*" user_name_attr: "uid" group_member_attr: "memberOf" superuser_filter: "" data_profiler_filter: "" bind_user: "cn=Manager,dc=example,dc=com" bind_password: "insecure" basedn: "dc=example,dc=com" cacert: "/etc/ca/ldap_ca.crt" search_scope: "LEVEL" mesos: # Shipyard is not using this master: "" framework_name: "" task_cpu: "" task_memory: "" checkpoint: "" authenticate: "" kerberos: # Shipyard is not using this ccache: "" principal: "" reinit_frequency: "" kinit_path: "" keytab: "" github_enterprise: # Shipyard is not using this api_rev: v3 admin: hide_sensitive_variable_fields: "True" elasticsearch: # Shipyard is not using this elasticsearch_host: "" elasticsearch_log_id_template: "" elasticsearch_end_of_log_mark: "" kubernetes: # Shipyard is not using this (maybe future for spawning own workers) worker_container_repository: "" worker_container_tag: "" delete_worker_pods: "True" namespace: "default" airflow_configmap: "" dags_volume_subpath: "" dags_volume_claim: "" logs_volume_subpath: "" logs_volume_claim: "" git_repo: "" git_branch: "" git_user: "" git_password: "" git_subpath: "" git_sync_container_repository: "" git_sync_container_tag: "" git_sync_init_container_name: "" worker_service_account_name: "" image_pull_secrets: "" gcp_service_account_keys: "" in_cluster: "" kubernetes_secrets: #Shipyard is not using this # End of Airflow config options pod: mounts: dag_path: /home/ubuntu/workbench/dags plugin_path: /home/ubuntu/workbench/plugins log_path: /home/ubuntu/workbench/logs airflow_flower: init_container: null airflow_flower: airflow_scheduler: init_container: null airflow_scheduler: airflow_worker: init_container: null airflow_worker: airflow_web: init_container: null airflow_web: shipyard: init_container: null shipyard: shipyard_db_init: init_container: null shipyard_db_init: shipyard_db_sync: init_container: null shipyard_db_sync: replicas: shipyard: api: 2 airflow: web: 2 worker: 2 flower: 2 scheduler: 2 lifecycle: upgrades: deployments: revision_history: 3 pod_replacement_strategy: RollingUpdate rolling_update: max_unavailable: 1 max_surge: 3 termination_grace_period: airflow: timeout: 30 shipyard: timeout: 30 resources: enabled: false airflow: flower: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" logrotate: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" scheduler: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" web: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" worker: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" shipyard_api: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" jobs: rabbit_init: limits: memory: "128Mi" cpu: "500m" requests: memory: "128Mi" cpu: "500m" airflow_db_init: limits: memory: "128Mi" cpu: "500m" requests: memory: "128Mi" cpu: "500m" airflow_db_sync: limits: memory: "128Mi" cpu: "500m" requests: memory: "128Mi" cpu: "500m" ks_endpoints: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" ks_service: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" ks_user: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" shipyard_db_init: limits: memory: "128Mi" cpu: "500m" requests: memory: "128Mi" cpu: "500m" shipyard_db_sync: limits: memory: "128Mi" cpu: "500m" requests: memory: "128Mi" cpu: "500m" test: shipyard: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" airflow: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" manifests: configmap_shipyard_bin: true configmap_shipyard_etc: true configmap_airflow_bin: true configmap_airflow_etc: true deployment_airflow_flower: true deployment_airflow_scheduler: true deployment_shipyard: true deployment_airflow_web: true statefulset_airflow_worker: true ingress_airflow_api: true ingress_shipyard_api: true job_shipyard_db_init: true job_shipyard_db_sync: true job_rabbit_init: true job_airflow_db_init: true job_airflow_db_sync: true job_ks_endpoints: true job_ks_service: true job_ks_user: true secret_airflow_db: true secret_shipyard_db: true secret_ingress_tls: true secret_keystone: true secret_rabbitmq: true service_airflow_ingress: true service_airflow_flower: true service_shipyard: true service_shipyard_ingress: true service_airflow_web: true service_airflow_worker: true service_discovery_airflow_worker: true test_shipyard_api: true