Move airflow scheduler to worker statefulset

Moves the airflow sceduler to a container in the airflow-worker
statefulset so that its version lifecycle matches that of the worker.
Leaves the stand-alone scheduler in place to support upgradability from
prior installations that included a standalone scheduler. New
installations are advised to turn off the scheduler template from
rendering using the values.yaml flag.

This is an attempt to make disruptive upgrades to airflow less impactful
to a "update_site" action from Shipyard.

Additionally this removes the template for airflow-flower, which is not in use.

Change-Id: I0608793ee6aba1eb3ce0f5e9567655287014a0ca
This commit is contained in:
Bryan Strassner 2019-01-07 14:17:12 -06:00
parent 6bd02eea84
commit 6b75c7119a
6 changed files with 74 additions and 208 deletions

View File

@ -1,74 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.manifests.deployment_airflow_flower }}
{{- $envAll := . }}
{{- $serviceAccountName := "airflow-flower" }}
{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
{{- $mounts_airflow_flower := .Values.pod.mounts.airflow_flower.airflow_flower }}
{{- $mounts_airflow_flower_init := .Values.pod.mounts.airflow_flower.init_container }}
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: airflow-flower
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
replicas: {{ .Values.pod.replicas.airflow.flower }}
{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
template:
metadata:
labels:
{{ tuple $envAll "airflow" "flower" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }}
spec:
serviceAccountName: {{ $serviceAccountName }}
nodeSelector:
{{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }}
restartPolicy: Always
terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }}
initContainers:
{{ tuple $envAll "airflow_server" $mounts_airflow_flower_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: airflow-flower
image: {{ .Values.images.tags.airflow }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.airflow.flower | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
# To prevent the error: ValueError: invalid literal for int() with base 10: 'tcp://10.0.0.83:5555'
- name: FLOWER_PORT
value: "5555"
ports:
- containerPort: {{ .Values.network.airflow.flower.port }}
args: ["flower"]
readinessProbe:
tcpSocket:
port: {{ .Values.network.airflow.flower.port }}
volumeMounts:
- name: airflow-etc
mountPath: {{ .Values.conf.airflow_config_file.path }}
subPath: airflow.cfg
readOnly: true
{{ if $mounts_airflow_flower.volumeMounts }}{{ toYaml $mounts_airflow_flower.volumeMounts | indent 12 }}{{ end }}
volumes:
- name: airflow-etc
configMap:
name: airflow-etc
defaultMode: 0444
{{ if $mounts_airflow_flower.volumes }}{{ toYaml $mounts_airflow_flower.volumes | indent 8 }}{{ end }}
{{- end }}

View File

@ -13,6 +13,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# NOTE: This is a "standalone" scheduler definition.
# TODO: Remove this template after it is no longer needed. Until the worker
# statefulset contains the scheduler, this scheduler pod needs to be
# available to provide continuity of a software/site/update. Once
# Shipyard/Airflow is running with the Scheduler/Worker combo, this
# scheduler pod can be disabled by turning off the
# manifests.deployment_airflow_scheduler value.
#
{{- if .Values.manifests.deployment_airflow_scheduler }}
{{- $envAll := . }}
{{- $serviceAccountName := "airflow-scheduler" }}

View File

@ -1,44 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.service_airflow_flower }}
{{- $envAll := . }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ tuple "airflow_flower" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
spec:
ports:
{{ if .Values.network.airflow.flower.enable_node_port }}
- name: http
nodePort: {{ .Values.network.airflow.flower.node_port }}
port: {{ .Values.network.airflow.flower.port }}
protocol: TCP
targetPort: {{ .Values.network.airflow.flower.port }}
{{ else }}
- name: http
port: {{ .Values.network.airflow.flower.port }}
protocol: TCP
targetPort: {{ .Values.network.airflow.flower.port }}
{{ end }}
selector:
{{ tuple $envAll "airflow" "flower" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
{{ if .Values.network.airflow.flower.enable_node_port }}
type: NodePort
{{ end }}
{{- end }}

View File

@ -111,6 +111,40 @@ spec:
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
{{- end }}
containers:
- name: airflow-scheduler
image: {{ .Values.images.tags.airflow }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.airflow.scheduler | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: AIRFLOW_CONN_AIRFLOWS_OWN_DB
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_URI
# Set to -1 to stop scheduler from going into crash loops
args: ["scheduler", "-n", "-1" ]
volumeMounts:
- name: airflow-etc
mountPath: {{ .Values.conf.airflow_config_file.path }}
subPath: airflow.cfg
readOnly: true
{{- if .Values.prod_environment }}
- name: shipyard-etc
mountPath: /usr/local/airflow/plugins/shipyard.conf
subPath: shipyard.conf
readOnly: true
- name: pod-shipyard-share-airflow-dags
mountPath: {{ .Values.conf.airflow.core.dags_folder }}
- name: pod-shipyard-share-airflow-plugins
mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
{{ else }}
- name: airflow-dags
mountPath: {{ .Values.conf.airflow.core.dags_folder }}
- name: airflow-plugins
mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
{{- end }}
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
- name: airflow-worker
image: {{ .Values.images.tags.airflow }}
imagePullPolicy: {{ .Values.images.pull_policy }}

View File

@ -71,10 +71,6 @@ network:
ingress:
public: true
proxy_read_timeout: 600
flower:
name: airflow-flower
port: 5555
enable_node_port: false
web:
name: airflow-web
port: 8080
@ -94,8 +90,6 @@ dependencies:
services:
- service: postgresql_shipyard_db
endpoint: internal
- service: airflow_flower
endpoint: internal
- service: airflow_web
endpoint: internal
shipyard_db_sync:
@ -140,8 +134,6 @@ dependencies:
- shipyard-ks-user
- shipyard-ks-endpoints
services:
- service: airflow_flower
endpoint: internal
- service: airflow_web
endpoint: internal
- service: identity
@ -239,19 +231,6 @@ endpoints:
default: http
host_fqdn_override:
default: null
airflow_flower:
name: airflow-flower
hosts:
default: airflow-flower
port:
airflow_flower:
default: 5555
path:
default: /
scheme:
default: http
host_fqdn_override:
default: null
airflow_worker:
name: airflow-worker
hosts:
@ -725,15 +704,14 @@ pod:
dag_path: /home/ubuntu/workbench/dags
plugin_path: /home/ubuntu/workbench/plugins
log_path: /home/ubuntu/workbench/logs
airflow_flower:
init_container: null
airflow_flower:
airflow_scheduler:
init_container: null
# TODO: This is only used if the standalone scheduler is enabled.
airflow_scheduler:
init_container: null
airflow_worker:
init_container: null
airflow_worker:
airflow_scheduler:
airflow_web:
init_container: null
airflow_web:
@ -752,7 +730,6 @@ pod:
airflow:
web: 2
worker: 2
flower: 2
scheduler: 2
lifecycle:
upgrades:
@ -770,13 +747,6 @@ pod:
resources:
enabled: false
airflow:
flower:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
logrotate:
limits:
memory: "128Mi"
@ -890,7 +860,8 @@ manifests:
configmap_shipyard_etc: true
configmap_airflow_bin: true
configmap_airflow_etc: true
deployment_airflow_flower: true
# TODO: Set this to false only if a new deployment, or if the worker pod is
# running the scheduler
deployment_airflow_scheduler: true
deployment_shipyard: true
deployment_airflow_web: true
@ -911,7 +882,6 @@ manifests:
secret_keystone: true
secret_rabbitmq: true
service_airflow_ingress: true
service_airflow_flower: true
service_shipyard: true
service_shipyard_ingress: true
service_airflow_web: true

View File

@ -39,68 +39,39 @@ developers to bring up Shipyard and the rest of the Airship components on a
single Ubuntu Virtual Machine.
The deployment is fully automated and can take a while to complete. It can take
30 minutes to an hour for a full deployment to complete.
30 minutes to an hour or more for a full deployment to complete.
Post Deployment
---------------
#. The environment should resemble the following after executing the required
#. The environment should include the following after executing the required
steps::
# sudo kubectl get pods -n ucp
# sudo kubectl get pods -n ucp | grep -v Completed
NAME READY STATUS RESTARTS AGE
airflow-flower-6cdc6f9cb4-5r62v 1/1 Running 0 3h
airflow-scheduler-6d54445bf8-6ldrd 1/1 Running 0 3h
airflow-web-7bd69d857d-qlptj 1/1 Running 0 3h
airflow-worker-666696d6c5-vffpg 1/1 Running 0 3h
armada-api-84df5b7fc9-4nxp5 1/1 Running 0 4h
barbican-api-85c956c84f-p4q7h 1/1 Running 0 4h
deckhand-5468d59455-2mcqd 1/1 Running 0 4h
drydock-api-f9897cf44-csbc8 1/1 Running 0 4h
drydock-api-f9897cf44-jgv4q 1/1 Running 0 4h
etcd-5bcbbd679c-rb5rf 1/1 Running 0 4h
ingress-api-xvkzx 1/1 Running 0 4h
ingress-error-pages-5d79688f6c-9b8xc 1/1 Running 0 4h
keystone-api-6bc85c98-886mg 1/1 Running 0 4h
maas-rack-5d4b84c4d5-dt87j 1/1 Running 0 4h
maas-region-0 1/1 Running 0 4h
mariadb-0 1/1 Running 0 4h
mariadb-1 1/1 Running 0 4h
mariadb-2 1/1 Running 0 4h
memcached-5bf49657db-kq6qh 1/1 Running 0 4h
postgresql-0 1/1 Running 0 4h
rabbitmq-f68649644-pnw6p 1/1 Running 0 4h
shipyard-6f4c7765d-n2kx6 1/1 Running 0 3h
#. Check that all relevant helm charts have been deployed::
# sudo helm ls
NAME REVISION UPDATED STATUS CHART NAMESPACE
ucp-armada 1 Fri Dec 1 10:03:44 2017 DEPLOYED armada-0.1.0 ucp
ucp-barbican 1 Fri Dec 1 10:03:47 2017 DEPLOYED barbican-0.1.0 ucp
ucp-calico 1 Fri Dec 1 10:00:05 2017 DEPLOYED calico-0.1.0 kube-system
ucp-calico-etcd 1 Fri Dec 1 09:59:28 2017 DEPLOYED etcd-0.1.0 kube-system
ucp-ceph 1 Fri Dec 1 10:00:58 2017 DEPLOYED ceph-0.1.0 ceph
ucp-coredns 1 Fri Dec 1 10:00:26 2017 DEPLOYED coredns-0.1.0 kube-system
ucp-deckhand 1 Fri Dec 1 10:03:39 2017 DEPLOYED deckhand-0.1.0 ucp
ucp-drydock 1 Fri Dec 1 10:03:37 2017 DEPLOYED drydock-0.1.0 ucp
ucp-etcd-rabbitmq 1 Fri Dec 1 10:02:44 2017 DEPLOYED etcd-0.1.0 ucp
ucp-ingress 1 Fri Dec 1 10:02:45 2017 DEPLOYED ingress-0.1.0 ucp
ucp-keystone 1 Fri Dec 1 10:03:45 2017 DEPLOYED keystone-0.1.0 ucp
ucp-kubernetes-apiserver 1 Fri Dec 1 10:00:32 2017 DEPLOYED apiserver-0.1.0 kube-system
ucp-kubernetes-controller-manager 1 Fri Dec 1 10:00:33 2017 DEPLOYED controller_manager-0.1.0 kube-system
ucp-kubernetes-etcd 1 Fri Dec 1 10:00:31 2017 DEPLOYED etcd-0.1.0 kube-system
ucp-kubernetes-proxy 1 Fri Dec 1 09:58:46 2017 DEPLOYED proxy-0.1.0 kube-system
ucp-kubernetes-scheduler 1 Fri Dec 1 10:00:34 2017 DEPLOYED scheduler-0.1.0 kube-system
ucp-maas 1 Fri Dec 1 10:03:36 2017 DEPLOYED maas-0.1.0 ucp
ucp-maas-postgresql 1 Fri Dec 1 10:02:44 2017 DEPLOYED postgresql-0.1.0 ucp
ucp-rabbitmq 1 Fri Dec 1 10:02:45 2017 DEPLOYED rabbitmq-0.1.0 ucp
ucp-rbac 1 Fri Dec 1 10:00:44 2017 DEPLOYED rbac-0.1.0 kube-system
ucp-shipyard 1 Fri Dec 1 10:38:08 2017 DEPLOYED shipyard-0.1.0 ucp
ucp-ucp-ceph-config 1 Fri Dec 1 10:02:40 2017 DEPLOYED ceph-0.1.0 ucp
ucp-ucp-mariadb 1 Fri Dec 1 10:02:43 2017 DEPLOYED mariadb-0.1.0 ucp
ucp-ucp-memcached 1 Fri Dec 1 10:02:44 2017 DEPLOYED memcached-0.1.0 ucp
airflow-scheduler-79754bfdd5-2wpxn 1/1 Running 0 4m
airflow-web-7679866685-g99qm 1/1 Running 0 4m
airflow-worker-0 3/3 Running 0 4m
airship-ucp-keystone-memcached-mem... 1/1 Running 0 31m
airship-ucp-rabbitmq-rabbitmq-0 1/1 Running 0 35m
armada-api-5488cbdb99-zjb8n 1/1 Running 0 12m
barbican-api-5fc8f7d6f-s7h7j 1/1 Running 0 11m
deckhand-api-7b476d6c46-qlvtm 1/1 Running 0 8m
drydock-api-5f9fdc858d-lnxvj 1/1 Running 0 1m
ingress-6cd5b89d5d-hzfzj 1/1 Running 0 35m
ingress-error-pages-5c97bb46bb-zqqbx 1/1 Running 0 35m
keystone-api-7657986b8c-6bf92 1/1 Running 0 31m
maas-ingress-66447d7445-mgklj 2/2 Running 0 27m
maas-ingress-errors-8686d56d98-vrjzg 1/1 Running 0 27m
maas-rack-0 1/1 Running 0 27m
maas-region-0 2/2 Running 0 27m
mariadb-ingress-6c4f9c76f-lk9ff 1/1 Running 0 35m
mariadb-ingress-6c4f9c76f-ns5kj 1/1 Running 0 35m
mariadb-ingress-error-pages-5dd6fb... 1/1 Running 0 35m
mariadb-server-0 1/1 Running 0 35m
postgresql-0 1/1 Running 0 32m
promenade-api-764b765d77-ffhv4 1/1 Running 0 7m
shipyard-api-69888d9f68-8ljfk 1/1 Running 0 4m
.. _airship-in-a-bottle: https://git.airshipit.org/cgit/airship-in-a-bottle
.. _dev_minimal: https://git.airshipit.org/cgit/airship-in-a-bottle/tree/manifests/dev_minimal