From 6b75c7119a850ee7f0c0651021076dfcf220e0f7 Mon Sep 17 00:00:00 2001 From: Bryan Strassner Date: Mon, 7 Jan 2019 14:17:12 -0600 Subject: [PATCH] Move airflow scheduler to worker statefulset Moves the airflow sceduler to a container in the airflow-worker statefulset so that its version lifecycle matches that of the worker. Leaves the stand-alone scheduler in place to support upgradability from prior installations that included a standalone scheduler. New installations are advised to turn off the scheduler template from rendering using the values.yaml flag. This is an attempt to make disruptive upgrades to airflow less impactful to a "update_site" action from Shipyard. Additionally this removes the template for airflow-flower, which is not in use. Change-Id: I0608793ee6aba1eb3ce0f5e9567655287014a0ca --- .../templates/deployment-airflow-flower.yaml | 74 ----------------- .../deployment-airflow-scheduler.yaml | 9 +++ .../templates/service-airflow-flower.yaml | 44 ---------- .../templates/statefulset-airflow-worker.yaml | 34 ++++++++ charts/shipyard/values.yaml | 40 ++------- doc/source/deployment-guide.rst | 81 ++++++------------- 6 files changed, 74 insertions(+), 208 deletions(-) delete mode 100644 charts/shipyard/templates/deployment-airflow-flower.yaml delete mode 100644 charts/shipyard/templates/service-airflow-flower.yaml diff --git a/charts/shipyard/templates/deployment-airflow-flower.yaml b/charts/shipyard/templates/deployment-airflow-flower.yaml deleted file mode 100644 index 8efecfae..00000000 --- a/charts/shipyard/templates/deployment-airflow-flower.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.manifests.deployment_airflow_flower }} -{{- $envAll := . }} -{{- $serviceAccountName := "airflow-flower" }} -{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} -{{- $mounts_airflow_flower := .Values.pod.mounts.airflow_flower.airflow_flower }} -{{- $mounts_airflow_flower_init := .Values.pod.mounts.airflow_flower.init_container }} ---- -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: airflow-flower - annotations: - {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} -spec: - replicas: {{ .Values.pod.replicas.airflow.flower }} -{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} - template: - metadata: - labels: -{{ tuple $envAll "airflow" "flower" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - annotations: - configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }} - configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }} - spec: - serviceAccountName: {{ $serviceAccountName }} - nodeSelector: - {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }} - restartPolicy: Always - terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} - initContainers: -{{ tuple $envAll "airflow_server" $mounts_airflow_flower_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: - - name: airflow-flower - image: {{ .Values.images.tags.airflow }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.airflow.flower | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - env: - # To prevent the error: ValueError: invalid literal for int() with base 10: 'tcp://10.0.0.83:5555' - - name: FLOWER_PORT - value: "5555" - ports: - - containerPort: {{ .Values.network.airflow.flower.port }} - args: ["flower"] - readinessProbe: - tcpSocket: - port: {{ .Values.network.airflow.flower.port }} - volumeMounts: - - name: airflow-etc - mountPath: {{ .Values.conf.airflow_config_file.path }} - subPath: airflow.cfg - readOnly: true -{{ if $mounts_airflow_flower.volumeMounts }}{{ toYaml $mounts_airflow_flower.volumeMounts | indent 12 }}{{ end }} - volumes: - - name: airflow-etc - configMap: - name: airflow-etc - defaultMode: 0444 -{{ if $mounts_airflow_flower.volumes }}{{ toYaml $mounts_airflow_flower.volumes | indent 8 }}{{ end }} -{{- end }} diff --git a/charts/shipyard/templates/deployment-airflow-scheduler.yaml b/charts/shipyard/templates/deployment-airflow-scheduler.yaml index f38b5021..8c6c597f 100644 --- a/charts/shipyard/templates/deployment-airflow-scheduler.yaml +++ b/charts/shipyard/templates/deployment-airflow-scheduler.yaml @@ -13,6 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +# NOTE: This is a "standalone" scheduler definition. +# TODO: Remove this template after it is no longer needed. Until the worker +# statefulset contains the scheduler, this scheduler pod needs to be +# available to provide continuity of a software/site/update. Once +# Shipyard/Airflow is running with the Scheduler/Worker combo, this +# scheduler pod can be disabled by turning off the +# manifests.deployment_airflow_scheduler value. +# + {{- if .Values.manifests.deployment_airflow_scheduler }} {{- $envAll := . }} {{- $serviceAccountName := "airflow-scheduler" }} diff --git a/charts/shipyard/templates/service-airflow-flower.yaml b/charts/shipyard/templates/service-airflow-flower.yaml deleted file mode 100644 index e7edab39..00000000 --- a/charts/shipyard/templates/service-airflow-flower.yaml +++ /dev/null @@ -1,44 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. -Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.service_airflow_flower }} -{{- $envAll := . }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ tuple "airflow_flower" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -spec: - ports: - {{ if .Values.network.airflow.flower.enable_node_port }} - - name: http - nodePort: {{ .Values.network.airflow.flower.node_port }} - port: {{ .Values.network.airflow.flower.port }} - protocol: TCP - targetPort: {{ .Values.network.airflow.flower.port }} - {{ else }} - - name: http - port: {{ .Values.network.airflow.flower.port }} - protocol: TCP - targetPort: {{ .Values.network.airflow.flower.port }} - {{ end }} - selector: -{{ tuple $envAll "airflow" "flower" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} - {{ if .Values.network.airflow.flower.enable_node_port }} - type: NodePort - {{ end }} -{{- end }} diff --git a/charts/shipyard/templates/statefulset-airflow-worker.yaml b/charts/shipyard/templates/statefulset-airflow-worker.yaml index 9cf6c33c..a53473d7 100644 --- a/charts/shipyard/templates/statefulset-airflow-worker.yaml +++ b/charts/shipyard/templates/statefulset-airflow-worker.yaml @@ -111,6 +111,40 @@ spec: mountPath: {{ .Values.conf.airflow.core.base_log_folder }} {{- end }} containers: + - name: airflow-scheduler + image: {{ .Values.images.tags.airflow }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.airflow.scheduler | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql_airflow_db.user }} + key: AIRFLOW_DATABASE_URI + # Set to -1 to stop scheduler from going into crash loops + args: ["scheduler", "-n", "-1" ] + volumeMounts: + - name: airflow-etc + mountPath: {{ .Values.conf.airflow_config_file.path }} + subPath: airflow.cfg + readOnly: true +{{- if .Values.prod_environment }} + - name: shipyard-etc + mountPath: /usr/local/airflow/plugins/shipyard.conf + subPath: shipyard.conf + readOnly: true + - name: pod-shipyard-share-airflow-dags + mountPath: {{ .Values.conf.airflow.core.dags_folder }} + - name: pod-shipyard-share-airflow-plugins + mountPath: {{ .Values.conf.airflow.core.plugins_folder }} +{{ else }} + - name: airflow-dags + mountPath: {{ .Values.conf.airflow.core.dags_folder }} + - name: airflow-plugins + mountPath: {{ .Values.conf.airflow.core.plugins_folder }} +{{- end }} + - name: airflow-logs + mountPath: {{ .Values.conf.airflow.core.base_log_folder }} - name: airflow-worker image: {{ .Values.images.tags.airflow }} imagePullPolicy: {{ .Values.images.pull_policy }} diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 9885db4b..969ad4b3 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -71,10 +71,6 @@ network: ingress: public: true proxy_read_timeout: 600 - flower: - name: airflow-flower - port: 5555 - enable_node_port: false web: name: airflow-web port: 8080 @@ -94,8 +90,6 @@ dependencies: services: - service: postgresql_shipyard_db endpoint: internal - - service: airflow_flower - endpoint: internal - service: airflow_web endpoint: internal shipyard_db_sync: @@ -140,8 +134,6 @@ dependencies: - shipyard-ks-user - shipyard-ks-endpoints services: - - service: airflow_flower - endpoint: internal - service: airflow_web endpoint: internal - service: identity @@ -239,19 +231,6 @@ endpoints: default: http host_fqdn_override: default: null - airflow_flower: - name: airflow-flower - hosts: - default: airflow-flower - port: - airflow_flower: - default: 5555 - path: - default: / - scheme: - default: http - host_fqdn_override: - default: null airflow_worker: name: airflow-worker hosts: @@ -725,15 +704,14 @@ pod: dag_path: /home/ubuntu/workbench/dags plugin_path: /home/ubuntu/workbench/plugins log_path: /home/ubuntu/workbench/logs - airflow_flower: - init_container: null - airflow_flower: airflow_scheduler: - init_container: null + # TODO: This is only used if the standalone scheduler is enabled. airflow_scheduler: + init_container: null airflow_worker: init_container: null airflow_worker: + airflow_scheduler: airflow_web: init_container: null airflow_web: @@ -752,7 +730,6 @@ pod: airflow: web: 2 worker: 2 - flower: 2 scheduler: 2 lifecycle: upgrades: @@ -770,13 +747,6 @@ pod: resources: enabled: false airflow: - flower: - limits: - memory: "128Mi" - cpu: "100m" - requests: - memory: "128Mi" - cpu: "100m" logrotate: limits: memory: "128Mi" @@ -890,7 +860,8 @@ manifests: configmap_shipyard_etc: true configmap_airflow_bin: true configmap_airflow_etc: true - deployment_airflow_flower: true + # TODO: Set this to false only if a new deployment, or if the worker pod is + # running the scheduler deployment_airflow_scheduler: true deployment_shipyard: true deployment_airflow_web: true @@ -911,7 +882,6 @@ manifests: secret_keystone: true secret_rabbitmq: true service_airflow_ingress: true - service_airflow_flower: true service_shipyard: true service_shipyard_ingress: true service_airflow_web: true diff --git a/doc/source/deployment-guide.rst b/doc/source/deployment-guide.rst index 7367bfff..6251bb51 100644 --- a/doc/source/deployment-guide.rst +++ b/doc/source/deployment-guide.rst @@ -39,68 +39,39 @@ developers to bring up Shipyard and the rest of the Airship components on a single Ubuntu Virtual Machine. The deployment is fully automated and can take a while to complete. It can take -30 minutes to an hour for a full deployment to complete. +30 minutes to an hour or more for a full deployment to complete. Post Deployment --------------- -#. The environment should resemble the following after executing the required +#. The environment should include the following after executing the required steps:: - # sudo kubectl get pods -n ucp + # sudo kubectl get pods -n ucp | grep -v Completed NAME READY STATUS RESTARTS AGE - airflow-flower-6cdc6f9cb4-5r62v 1/1 Running 0 3h - airflow-scheduler-6d54445bf8-6ldrd 1/1 Running 0 3h - airflow-web-7bd69d857d-qlptj 1/1 Running 0 3h - airflow-worker-666696d6c5-vffpg 1/1 Running 0 3h - armada-api-84df5b7fc9-4nxp5 1/1 Running 0 4h - barbican-api-85c956c84f-p4q7h 1/1 Running 0 4h - deckhand-5468d59455-2mcqd 1/1 Running 0 4h - drydock-api-f9897cf44-csbc8 1/1 Running 0 4h - drydock-api-f9897cf44-jgv4q 1/1 Running 0 4h - etcd-5bcbbd679c-rb5rf 1/1 Running 0 4h - ingress-api-xvkzx 1/1 Running 0 4h - ingress-error-pages-5d79688f6c-9b8xc 1/1 Running 0 4h - keystone-api-6bc85c98-886mg 1/1 Running 0 4h - maas-rack-5d4b84c4d5-dt87j 1/1 Running 0 4h - maas-region-0 1/1 Running 0 4h - mariadb-0 1/1 Running 0 4h - mariadb-1 1/1 Running 0 4h - mariadb-2 1/1 Running 0 4h - memcached-5bf49657db-kq6qh 1/1 Running 0 4h - postgresql-0 1/1 Running 0 4h - rabbitmq-f68649644-pnw6p 1/1 Running 0 4h - shipyard-6f4c7765d-n2kx6 1/1 Running 0 3h - -#. Check that all relevant helm charts have been deployed:: - - # sudo helm ls - NAME REVISION UPDATED STATUS CHART NAMESPACE - ucp-armada 1 Fri Dec 1 10:03:44 2017 DEPLOYED armada-0.1.0 ucp - ucp-barbican 1 Fri Dec 1 10:03:47 2017 DEPLOYED barbican-0.1.0 ucp - ucp-calico 1 Fri Dec 1 10:00:05 2017 DEPLOYED calico-0.1.0 kube-system - ucp-calico-etcd 1 Fri Dec 1 09:59:28 2017 DEPLOYED etcd-0.1.0 kube-system - ucp-ceph 1 Fri Dec 1 10:00:58 2017 DEPLOYED ceph-0.1.0 ceph - ucp-coredns 1 Fri Dec 1 10:00:26 2017 DEPLOYED coredns-0.1.0 kube-system - ucp-deckhand 1 Fri Dec 1 10:03:39 2017 DEPLOYED deckhand-0.1.0 ucp - ucp-drydock 1 Fri Dec 1 10:03:37 2017 DEPLOYED drydock-0.1.0 ucp - ucp-etcd-rabbitmq 1 Fri Dec 1 10:02:44 2017 DEPLOYED etcd-0.1.0 ucp - ucp-ingress 1 Fri Dec 1 10:02:45 2017 DEPLOYED ingress-0.1.0 ucp - ucp-keystone 1 Fri Dec 1 10:03:45 2017 DEPLOYED keystone-0.1.0 ucp - ucp-kubernetes-apiserver 1 Fri Dec 1 10:00:32 2017 DEPLOYED apiserver-0.1.0 kube-system - ucp-kubernetes-controller-manager 1 Fri Dec 1 10:00:33 2017 DEPLOYED controller_manager-0.1.0 kube-system - ucp-kubernetes-etcd 1 Fri Dec 1 10:00:31 2017 DEPLOYED etcd-0.1.0 kube-system - ucp-kubernetes-proxy 1 Fri Dec 1 09:58:46 2017 DEPLOYED proxy-0.1.0 kube-system - ucp-kubernetes-scheduler 1 Fri Dec 1 10:00:34 2017 DEPLOYED scheduler-0.1.0 kube-system - ucp-maas 1 Fri Dec 1 10:03:36 2017 DEPLOYED maas-0.1.0 ucp - ucp-maas-postgresql 1 Fri Dec 1 10:02:44 2017 DEPLOYED postgresql-0.1.0 ucp - ucp-rabbitmq 1 Fri Dec 1 10:02:45 2017 DEPLOYED rabbitmq-0.1.0 ucp - ucp-rbac 1 Fri Dec 1 10:00:44 2017 DEPLOYED rbac-0.1.0 kube-system - ucp-shipyard 1 Fri Dec 1 10:38:08 2017 DEPLOYED shipyard-0.1.0 ucp - ucp-ucp-ceph-config 1 Fri Dec 1 10:02:40 2017 DEPLOYED ceph-0.1.0 ucp - ucp-ucp-mariadb 1 Fri Dec 1 10:02:43 2017 DEPLOYED mariadb-0.1.0 ucp - ucp-ucp-memcached 1 Fri Dec 1 10:02:44 2017 DEPLOYED memcached-0.1.0 ucp - + airflow-scheduler-79754bfdd5-2wpxn 1/1 Running 0 4m + airflow-web-7679866685-g99qm 1/1 Running 0 4m + airflow-worker-0 3/3 Running 0 4m + airship-ucp-keystone-memcached-mem... 1/1 Running 0 31m + airship-ucp-rabbitmq-rabbitmq-0 1/1 Running 0 35m + armada-api-5488cbdb99-zjb8n 1/1 Running 0 12m + barbican-api-5fc8f7d6f-s7h7j 1/1 Running 0 11m + deckhand-api-7b476d6c46-qlvtm 1/1 Running 0 8m + drydock-api-5f9fdc858d-lnxvj 1/1 Running 0 1m + ingress-6cd5b89d5d-hzfzj 1/1 Running 0 35m + ingress-error-pages-5c97bb46bb-zqqbx 1/1 Running 0 35m + keystone-api-7657986b8c-6bf92 1/1 Running 0 31m + maas-ingress-66447d7445-mgklj 2/2 Running 0 27m + maas-ingress-errors-8686d56d98-vrjzg 1/1 Running 0 27m + maas-rack-0 1/1 Running 0 27m + maas-region-0 2/2 Running 0 27m + mariadb-ingress-6c4f9c76f-lk9ff 1/1 Running 0 35m + mariadb-ingress-6c4f9c76f-ns5kj 1/1 Running 0 35m + mariadb-ingress-error-pages-5dd6fb... 1/1 Running 0 35m + mariadb-server-0 1/1 Running 0 35m + postgresql-0 1/1 Running 0 32m + promenade-api-764b765d77-ffhv4 1/1 Running 0 7m + shipyard-api-69888d9f68-8ljfk 1/1 Running 0 4m .. _airship-in-a-bottle: https://git.airshipit.org/cgit/airship-in-a-bottle .. _dev_minimal: https://git.airshipit.org/cgit/airship-in-a-bottle/tree/manifests/dev_minimal