From a11e962eef5a5aa8f8fc15c4a324dfa6b2465061 Mon Sep 17 00:00:00 2001 From: Bryan Strassner Date: Fri, 18 Jan 2019 15:50:31 -0600 Subject: [PATCH] Move Airflow web container into Shipyard pod Moves the airflow web server container from its own pod into the Shipyard pod. This removes exposed network surface area from the Shipyard suite of software. Shipyard, after this change accesses the Airflow API using localhost in the same k8s pod. Change-Id: Ied4bd415a8d78c393b7256ead27a6a2176f4a2d6 --- .../deployment-airflow-scheduler.yaml | 6 +- .../templates/deployment-airflow-web.yaml | 92 ------------------- .../templates/deployment-shipyard.yaml | 41 ++++++++- .../templates/ingress-airflow-api.yaml | 49 ---------- .../templates/service-airflow-ingress.yaml | 33 ------- .../templates/service-airflow-web.yaml | 44 --------- .../templates/statefulset-airflow-worker.yaml | 6 +- charts/shipyard/values.yaml | 56 ++--------- .../_static/shipyard.policy.yaml.sample | 13 ++- .../etc/shipyard/policy.yaml.sample | 13 ++- 10 files changed, 78 insertions(+), 275 deletions(-) delete mode 100644 charts/shipyard/templates/deployment-airflow-web.yaml delete mode 100644 charts/shipyard/templates/ingress-airflow-api.yaml delete mode 100644 charts/shipyard/templates/service-airflow-ingress.yaml delete mode 100644 charts/shipyard/templates/service-airflow-web.yaml diff --git a/charts/shipyard/templates/deployment-airflow-scheduler.yaml b/charts/shipyard/templates/deployment-airflow-scheduler.yaml index ff278aa2..c8fccf11 100644 --- a/charts/shipyard/templates/deployment-airflow-scheduler.yaml +++ b/charts/shipyard/templates/deployment-airflow-scheduler.yaml @@ -61,9 +61,9 @@ spec: env: - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql_airflow_db.user }} - key: AIRFLOW_DATABASE_URI + secretKeyRef: + name: {{ .Values.secrets.postgresql_airflow_db.user }} + key: AIRFLOW_DATABASE_URI # Set to -1 to stop scheduler from going into crash loops args: ["scheduler", "-n", "-1" ] volumeMounts: diff --git a/charts/shipyard/templates/deployment-airflow-web.yaml b/charts/shipyard/templates/deployment-airflow-web.yaml deleted file mode 100644 index 648f5564..00000000 --- a/charts/shipyard/templates/deployment-airflow-web.yaml +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.manifests.deployment_airflow_web }} -{{- $envAll := . }} -{{- $serviceAccountName := "airflow-web" }} -{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} -{{- $mounts_airflow_web := .Values.pod.mounts.airflow_web.airflow_web }} -{{- $mounts_airflow_web_init := .Values.pod.mounts.airflow_web.init_container }} ---- -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: airflow-web - annotations: - {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} -spec: - replicas: {{ .Values.pod.replicas.airflow.web }} -{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} - template: - metadata: - labels: -{{ tuple $envAll "airflow" "web" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - annotations: - configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }} - configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }} - spec: - serviceAccountName: {{ $serviceAccountName }} - nodeSelector: - {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }} - restartPolicy: Always - terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} - initContainers: -{{ tuple $envAll "airflow_server" $mounts_airflow_web_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: - - name: airflow-web - image: {{ .Values.images.tags.airflow }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.airflow.web | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - env: - - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql_airflow_db.user }} - key: AIRFLOW_DATABASE_URI - ports: - - containerPort: {{ .Values.network.airflow.web.port }} - args: ["webserver"] - readinessProbe: - tcpSocket: - port: {{ .Values.network.airflow.web.port }} - volumeMounts: - - name: airflow-etc - mountPath: {{ .Values.conf.airflow_config_file.path }} - subPath: airflow.cfg - readOnly: true - - name: shipyard-etc - mountPath: /usr/local/airflow/plugins/shipyard.conf - subPath: shipyard.conf - readOnly: true - - name: airflow-logs - mountPath: {{ .Values.conf.airflow.core.base_log_folder }} -{{ if $mounts_airflow_web.volumeMounts }}{{ toYaml $mounts_airflow_web.volumeMounts | indent 12 }}{{ end }} - volumes: - - name: airflow-etc - configMap: - name: airflow-etc - defaultMode: 0444 - - name: shipyard-etc - configMap: - name: shipyard-etc - defaultMode: 0444 - - name: airflow-bin - configMap: - name: airflow-bin - defaultMode: 0555 - - name: airflow-logs - emptyDir: {} -{{ if $mounts_airflow_web.volumes }}{{ toYaml $mounts_airflow_web.volumes | indent 8 }}{{ end }} -{{- end }} diff --git a/charts/shipyard/templates/deployment-shipyard.yaml b/charts/shipyard/templates/deployment-shipyard.yaml index 2d2700c3..a8cb104c 100644 --- a/charts/shipyard/templates/deployment-shipyard.yaml +++ b/charts/shipyard/templates/deployment-shipyard.yaml @@ -15,9 +15,10 @@ {{- if .Values.manifests.deployment_shipyard }} {{- $envAll := . }} +{{- $serviceAccountName := "shipyard" }} {{- $mounts_shipyard := .Values.pod.mounts.shipyard.shipyard }} {{- $mounts_shipyard_init := .Values.pod.mounts.shipyard.init_container }} -{{- $serviceAccountName := "shipyard" }} + {{ tuple $envAll "shipyard" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: apps/v1beta1 @@ -34,8 +35,10 @@ spec: labels: {{ tuple $envAll "shipyard" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} annotations: - configmap-bin-hash: {{ tuple "configmap-shipyard-bin.yaml" . | include "helm-toolkit.utils.hash" }} - configmap-etc-hash: {{ tuple "configmap-shipyard-etc.yaml" . | include "helm-toolkit.utils.hash" }} + shipyard-configmap-bin-hash: {{ tuple "configmap-shipyard-bin.yaml" . | include "helm-toolkit.utils.hash" }} + shipyard-configmap-etc-hash: {{ tuple "configmap-shipyard-etc.yaml" . | include "helm-toolkit.utils.hash" }} + airflow-configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }} + airflow-configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }} spec: serviceAccountName: {{ $serviceAccountName }} nodeSelector: @@ -88,6 +91,28 @@ spec: - name: tmp-profiles mountPath: /tmp/profiles {{ end }} + - name: airflow-web + image: {{ .Values.images.tags.airflow }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.airflow.web | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql_airflow_db.user }} + key: AIRFLOW_DATABASE_URI + args: ["webserver"] + volumeMounts: + - name: airflow-etc + mountPath: {{ .Values.conf.airflow_config_file.path }} + subPath: airflow.cfg + readOnly: true + - name: shipyard-etc + mountPath: /usr/local/airflow/plugins/shipyard.conf + subPath: shipyard.conf + readOnly: true + - name: airflow-logs + mountPath: {{ .Values.conf.airflow.core.base_log_folder }} {{ if $mounts_shipyard.volumeMounts }}{{ toYaml $mounts_shipyard.volumeMounts | indent 12 }}{{ end }} volumes: {{ if .Values.conf.shipyard.base.profiler }} @@ -100,5 +125,15 @@ spec: configMap: name: shipyard-etc defaultMode: 0444 + - name: airflow-etc + configMap: + name: airflow-etc + defaultMode: 0444 + - name: airflow-bin + configMap: + name: airflow-bin + defaultMode: 0555 + - name: airflow-logs + emptyDir: {} {{ if $mounts_shipyard.volumes }}{{ toYaml $mounts_shipyard.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/charts/shipyard/templates/ingress-airflow-api.yaml b/charts/shipyard/templates/ingress-airflow-api.yaml deleted file mode 100644 index f9c2deba..00000000 --- a/charts/shipyard/templates/ingress-airflow-api.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. -Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.ingress_airflow_api }} -{{- $envAll := . }} -{{- if .Values.network.airflow.ingress.public }} -{{- $backendServiceType := "airflow_web" }} -{{- $backendPort := "http" }} -{{- $ingressName := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -{{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -{{- $hostName := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -{{- $hostNameNamespaced := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }} -{{- $hostNameFull := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: {{ $ingressName }} - annotations: - kubernetes.io/ingress.class: "nginx" - ingress.kubernetes.io/rewrite-target: / - nginx.ingress.kubernetes.io/proxy-read-timeout: {{ .Values.network.airflow.ingress.proxy_read_timeout | quote }} -spec: - rules: -{{- range $key1, $vHost := tuple $hostName $hostNameNamespaced $hostNameFull }} - - host: {{ $vHost }} - http: - paths: - - path: / - backend: - serviceName: {{ $backendName }} - servicePort: {{ $backendPort }} -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/shipyard/templates/service-airflow-ingress.yaml b/charts/shipyard/templates/service-airflow-ingress.yaml deleted file mode 100644 index a3989067..00000000 --- a/charts/shipyard/templates/service-airflow-ingress.yaml +++ /dev/null @@ -1,33 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. -Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.service_airflow_ingress }} -{{- $envAll := . }} -{{- if .Values.network.airflow.ingress.public }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ tuple "airflow_web" "public" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -spec: - ports: - - name: http - port: 80 - selector: - app: ingress-api -{{- end }} -{{- end }} diff --git a/charts/shipyard/templates/service-airflow-web.yaml b/charts/shipyard/templates/service-airflow-web.yaml deleted file mode 100644 index bb68fe60..00000000 --- a/charts/shipyard/templates/service-airflow-web.yaml +++ /dev/null @@ -1,44 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. -Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.service_airflow_web }} -{{- $envAll := . }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ tuple "airflow_web" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -spec: - ports: - {{ if .Values.network.airflow.web.enable_node_port }} - - name: http - nodePort: {{ .Values.network.airflow.web.node_port }} - port: {{ .Values.network.airflow.web.port }} - protocol: TCP - targetPort: {{ .Values.network.airflow.web.port }} - {{ else }} - - name: http - port: {{ .Values.network.airflow.web.port }} - protocol: TCP - targetPort: {{ .Values.network.airflow.web.port }} - {{ end }} - selector: -{{ tuple $envAll "airflow" "web" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} - {{ if .Values.network.airflow.web.enable_node_port }} - type: NodePort - {{ end }} -{{- end }} diff --git a/charts/shipyard/templates/statefulset-airflow-worker.yaml b/charts/shipyard/templates/statefulset-airflow-worker.yaml index c71b63eb..64a3997e 100644 --- a/charts/shipyard/templates/statefulset-airflow-worker.yaml +++ b/charts/shipyard/templates/statefulset-airflow-worker.yaml @@ -101,9 +101,9 @@ spec: env: - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql_airflow_db.user }} - key: AIRFLOW_DATABASE_URI + secretKeyRef: + name: {{ .Values.secrets.postgresql_airflow_db.user }} + key: AIRFLOW_DATABASE_URI # Set to -1 to stop scheduler from going into crash loops args: ["scheduler", "-n", "-1" ] volumeMounts: diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 998a4295..5e780180 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -65,14 +65,6 @@ network: node_port: 31901 enable_node_port: false airflow: - ingress: - public: true - proxy_read_timeout: 600 - web: - name: airflow-web - port: 8080 - node_port: 32080 - enable_node_port: false worker: name: airflow-worker port: 8793 @@ -87,8 +79,6 @@ dependencies: services: - service: postgresql_shipyard_db endpoint: internal - - service: airflow_web - endpoint: internal shipyard_db_sync: jobs: - shipyard-db-init @@ -131,8 +121,6 @@ dependencies: - shipyard-ks-user - shipyard-ks-endpoints services: - - service: airflow_web - endpoint: internal - service: identity endpoint: internal - service: postgresql_shipyard_db @@ -213,21 +201,6 @@ endpoints: # tls: # crt: null # key: null - airflow_web: - name: airflow-web - hosts: - default: airflow-web-int - public: airflow-web - port: - airflow_web: - default: 8080 - public: 80 - path: - default: / - scheme: - default: http - host_fqdn_override: - default: null airflow_worker: name: airflow-worker hosts: @@ -394,7 +367,7 @@ conf: paste.filter_factory: keystonemiddleware.auth_token:filter_factory shipyard: base: - web_server: + web_server: http://localhost:8080/ pool_size: 15 pool_pre_ping: true pool_timeout: 30 @@ -507,7 +480,9 @@ conf: worker_precheck: "False" cli: api_client: airflow.api.client.local_client - # endpoint_url is extracted from endpoints by the configmap template + # if endpoint_url is not set, it is extracted from endpoints by the + # configmap template + endpoint_url: http://localhost/ api: auth_backend: airflow.api.auth.backend.default lineage: @@ -530,8 +505,12 @@ conf: # Shipyard is not using this default_hive_mapred_queue: "" webserver: - # base_url is extracted from endpoints by the configmap template - web_server_host: 0.0.0.0 + # if base_url is not set, is extracted from endpoints by the configmap + # template + base_url: http://localhost/ + # set web_server_host to 0.0.0.0 to bind to all interfaces. By default + # only bind to loopback + web_server_host: 127.0.0.1 web_server_port: 8080 web_server_ssl_cert: "" web_server_ssl_key: "" @@ -706,9 +685,6 @@ pod: init_container: null airflow_worker: airflow_scheduler: - airflow_web: - init_container: null - airflow_web: shipyard: init_container: null shipyard: @@ -722,7 +698,6 @@ pod: shipyard: api: 2 airflow: - web: 2 worker: 2 scheduler: 2 lifecycle: @@ -841,13 +816,6 @@ pod: requests: memory: "128Mi" cpu: "100m" - airflow: - limits: - memory: "128Mi" - cpu: "100m" - requests: - memory: "128Mi" - cpu: "100m" manifests: configmap_shipyard_bin: true @@ -858,9 +826,7 @@ manifests: # running the scheduler deployment_airflow_scheduler: true deployment_shipyard: true - deployment_airflow_web: true statefulset_airflow_worker: true - ingress_airflow_api: true ingress_shipyard_api: true job_shipyard_db_init: true job_shipyard_db_sync: true @@ -875,10 +841,8 @@ manifests: secret_ingress_tls: true secret_keystone: true secret_rabbitmq: true - service_airflow_ingress: true service_shipyard: true service_shipyard_ingress: true - service_airflow_web: true service_airflow_worker: true service_discovery_airflow_worker: true test_shipyard_api: true diff --git a/doc/source/_static/shipyard.policy.yaml.sample b/doc/source/_static/shipyard.policy.yaml.sample index 1692ac5b..63a4c9c7 100644 --- a/doc/source/_static/shipyard.policy.yaml.sample +++ b/doc/source/_static/shipyard.policy.yaml.sample @@ -40,10 +40,16 @@ # POST /api/v1.0/configdocs/{collection_id} #"workflow_orchestrator:create_configdocs": "rule:admin_required" -# Retrieve a collection of configuration documents +# Retrieve a collection of configuration documents with redacted +# secrets # GET /api/v1.0/configdocs/{collection_id} #"workflow_orchestrator:get_configdocs": "rule:admin_required" +# Retrieve a collection of configuration documents with cleartext +# secrets. +# GET /api/v1.0/configdocs/{collection_id} +#"workflow_orchestrator:get_configdocs_cleartext": "rule:admin_required" + # Move documents from the Shipyard buffer to the committed documents # POST /api/v1.0/commitconfigdocs #"workflow_orchestrator:commit_configdocs": "rule:admin_required" @@ -53,6 +59,11 @@ # GET /api/v1.0/renderedconfigdocs #"workflow_orchestrator:get_renderedconfigdocs": "rule:admin_required" +# Retrieve the configuration documents with cleartext secrets rendered +# by Deckhand into a complete design +# GET /api/v1.0/renderedconfigdocs +#"workflow_orchestrator:get_renderedconfigdocs_cleartext": "rule:admin_required" + # Retrieve the list of workflows (DAGs) that have been invoked in # Airflow, whether via Shipyard or scheduled # GET /api/v1.0/workflows diff --git a/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample b/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample index 1692ac5b..63a4c9c7 100644 --- a/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample +++ b/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample @@ -40,10 +40,16 @@ # POST /api/v1.0/configdocs/{collection_id} #"workflow_orchestrator:create_configdocs": "rule:admin_required" -# Retrieve a collection of configuration documents +# Retrieve a collection of configuration documents with redacted +# secrets # GET /api/v1.0/configdocs/{collection_id} #"workflow_orchestrator:get_configdocs": "rule:admin_required" +# Retrieve a collection of configuration documents with cleartext +# secrets. +# GET /api/v1.0/configdocs/{collection_id} +#"workflow_orchestrator:get_configdocs_cleartext": "rule:admin_required" + # Move documents from the Shipyard buffer to the committed documents # POST /api/v1.0/commitconfigdocs #"workflow_orchestrator:commit_configdocs": "rule:admin_required" @@ -53,6 +59,11 @@ # GET /api/v1.0/renderedconfigdocs #"workflow_orchestrator:get_renderedconfigdocs": "rule:admin_required" +# Retrieve the configuration documents with cleartext secrets rendered +# by Deckhand into a complete design +# GET /api/v1.0/renderedconfigdocs +#"workflow_orchestrator:get_renderedconfigdocs_cleartext": "rule:admin_required" + # Retrieve the list of workflows (DAGs) that have been invoked in # Airflow, whether via Shipyard or scheduled # GET /api/v1.0/workflows