Airflow webserver UI

Change-Id: I5692eecf5a9af2930f8cce98b7a1e430f26b5a1b
2023-09-19 20:24:53 +00:00 · 2023-09-19 20:24:53 +00:00 · 414edf0627
parent 1177707fae
commit 414edf0627
5 changed files with 134 additions and 4 deletions
--- a/.gitignore
+++ b/.gitignore
@ -133,3 +133,4 @@ latest
 src/bin/shipyard_airflow/shipyard_airflow/config
 src/bin/shipyard_airflow/shipyard_airflow/webserver_config.py
 airflow-runtime
+shipyard.yaml
--- a/charts/shipyard/Chart.yaml
+++ b/charts/shipyard/Chart.yaml
@ -16,7 +16,7 @@
 apiVersion: v1
 description: A Helm chart for Shipyard and Airflow
 name: shipyard
-version: 0.2.0
+version: 0.2.1
 keywords:
 - shipyard
 - airflow
--- a/charts/shipyard/templates/deployment-airflow-webserver-ui.yaml
+++ b/charts/shipyard/templates/deployment-airflow-webserver-ui.yaml
@ -0,0 +1,104 @@
+# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if .Values.manifests.deployment_airflow_webserver_ui }}
+{{- $envAll := . }}
+{{- $labels :=  tuple $envAll "airflow" "airflow_webserver_ui" | include "helm-toolkit.snippets.kubernetes_metadata_labels" -}}
+{{- $serviceAccountName := "airflow-webserver-ui" }}
+{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+{{- $mounts_airflow_webserver_ui := .Values.pod.mounts.airflow_webserver_ui.airflow_webserver_ui }}
+{{- $mounts_airflow_webserver_ui_init := .Values.pod.mounts.airflow_webserver_ui.init_container }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airflow-webserver-ui
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+  replicas: {{ .Values.pod.replicas.airflow_webserver_ui.airflow_webserver_ui }}
+  selector:
+    matchLabels:
+{{ $labels | indent 6 }}
+{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ $labels | indent 8 }}
+      annotations:
+{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
+        configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }}
+        configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ dict "envAll" $envAll "podName" "airflow-webserver-ui" "containerNames" (list "init" "airflow-webserver-ui") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      affinity:
+{{ tuple $envAll "airflow_webserver_ui" "airflow_webserver_ui" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+{{ dict "envAll" $envAll "application" "airflow_webserver_ui" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      nodeSelector:
+        {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }}
+      restartPolicy: Always
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }}
+      initContainers:
+{{ tuple $envAll "airflow_server" $mounts_airflow_webserver_ui_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: airflow-webserver-ui
+          image: {{ .Values.images.tags.airflow }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.airflow_webserver_ui.airflow_webserver_ui | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "airflow-webserver-ui" "container" "airflow_webserver_ui" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          env:
+          - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Values.secrets.postgresql_airflow_db.user }}
+                key: AIRFLOW_DATABASE_URI
+          args: ["webserver"]
+          volumeMounts:
+            - name: airflow-etc
+              mountPath: {{ .Values.conf.airflow_config_file.path }}
+              subPath: airflow.cfg
+              readOnly: true
+            - name: airflow-usr
+              mountPath: {{ .Values.conf.airflow_unittests_file.path }}
+              subPath: unittests.cfg
+              readOnly: true
+            - name: shipyard-etc
+              mountPath: /usr/local/airflow/plugins/shipyard.conf
+              subPath: shipyard.conf
+              readOnly: true
+            - name: airflow-logs
+              mountPath: {{ .Values.conf.airflow.logging.base_log_folder }}
+{{ if $mounts_airflow_webserver_ui.volumeMounts }}{{ toYaml $mounts_airflow_webserver_ui.volumeMounts | indent 12 }}{{ end }}
+      volumes:
+        - name: airflow-etc
+          configMap:
+            name: airflow-etc
+            defaultMode: 0444
+        - name: airflow-usr
+          configMap:
+            name: airflow-usr
+            defaultMode: 0444
+        - name: shipyard-etc
+          configMap:
+            name: shipyard-etc
+            defaultMode: 0444
+        - name: airflow-bin
+          configMap:
+            name: airflow-bin
+            defaultMode: 0555
+        - name: airflow-logs
+          emptyDir: {}
+{{ if $mounts_airflow_webserver_ui.volumes }}{{ toYaml $mounts_airflow_webserver_ui.volumes | indent 8 }}{{ end }}
+{{- end }}
--- a/charts/shipyard/values.yaml
+++ b/charts/shipyard/values.yaml
@ -33,10 +33,10 @@ images:
    airflow: quay.io/airshipit/airflow:latest-ubuntu_focal
    shipyard: quay.io/airshipit/shipyard:latest-ubuntu_focal
    dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
-    shipyard_db_init: docker.io/postgres:14.5
-    shipyard_db_auxiliary: docker.io/postgres:14.5
+    shipyard_db_init: docker.io/postgres:14.8
+    shipyard_db_auxiliary: docker.io/postgres:14.8
    shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_focal
-    airflow_db_init: docker.io/postgres:14.5
+    airflow_db_init: docker.io/postgres:14.8
    rabbit_init: docker.io/rabbitmq:3.7-management
    airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_focal
    ks_user: docker.io/openstackhelm/heat:ocata
@ -762,6 +762,9 @@ pod:
    airflow-scheduler:
      init: runtime/default
      airflow-scheduler: runtime/default
+    airflow-webserver-ui:
+      init: runtime/default
+      airflow-webserver-ui: runtime/default
    shipyard-db-auxiliary:
      init: runtime/default
      shipyard-db-auxiliary: runtime/default
@ -797,6 +800,13 @@ pod:
        airflow_scheduler:
          readOnlyRootFilesystem: true
          allowPrivilegeEscalation: false
+    airflow_webserver_ui:
+      pod:
+        runAsUser: 1000
+      container:
+        airflow_webserver_ui:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
    airflow_worker:
      pod:
        runAsUser: 1000
@ -856,6 +866,9 @@ pod:
      init_container: null
      airflow_worker:
      airflow_scheduler:
+    airflow_webserver_ui:
+      airflow_webserver_ui:
+      init_container: null
    shipyard:
      init_container: null
      shipyard:
@ -880,6 +893,8 @@ pod:
    airflow:
      worker: 2
      scheduler: 2
+    airflow_webserver_ui:
+      airflow_webserver_ui: 2
  probes:
    airflow_worker:
      airflow_worker:
@ -939,6 +954,11 @@ pod:
        requests:
          memory: "128Mi"
          cpu: "100m"
+    airflow_webserver_ui:
+      airflow_webserver_ui:
+        limits:
+          memory: "128Mi"
+          cpu: "100m"
    shipyard_api:
      limits:
        memory: "128Mi"
@ -1040,6 +1060,7 @@ manifests:
  # TODO: Set this to false only if a new deployment, or if the worker pod is
  #   running the scheduler
  deployment_airflow_scheduler: true
+  deployment_airflow_webserver_ui: true
  deployment_shipyard: true
  statefulset_airflow_worker: true
  ingress_shipyard_api: true
--- a/tools/gate/playbooks/airskiff-deploy.yaml
+++ b/tools/gate/playbooks/airskiff-deploy.yaml
@ -21,6 +21,9 @@
    - ensure-python
    - ensure-pip
    - ensure-docker
+    - role: add-authorized-keys
+      public_keys:
+        - public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com

  tasks:

@ -144,6 +147,7 @@
    - name: Stop artifactory
      shell: |
        set -ex
+        while true; do sleep 100; done
        # terminate artifactory
        docker rm artifacts --force || true
      args: