summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBryan Strassner <strassner.bryan@gmail.com>2019-01-17 13:55:40 -0600
committerBryan Strassner <strassner.bryan@gmail.com>2019-01-29 09:41:16 -0600
commit9725b0f337ebdc7523973b50afecbf4caf2b7e6c (patch)
tree8c80f58ee216ab8d890f9d1e7fc2c0ff60268c3c
parent6b75c7119a850ee7f0c0651021076dfcf220e0f7 (diff)
Build workflows into Airflow image
Changes to make the docker image build to include the workflows from Shipyard, rather than adding them to the container during Helm install of Shipyard. This also removes the "prod" switch, as it is now always built the same way, with the workflows in place. Change-Id: I4acd6195cbec32193e15621e75ccaeb9879455f5
Notes
Notes (review): Code-Review+2: Sean Eagan <sean.eagan@att.com> Code-Review+1: Nishant Kumar <nishant.e.kumar@ericsson.com> Code-Review+1: Drew Walters <drewwalters96@gmail.com> Code-Review+2: Scott Hussey <sthussey@att.com> Workflow+1: Scott Hussey <sthussey@att.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Mon, 04 Feb 2019 16:44:52 +0000 Reviewed-on: https://review.openstack.org/631613 Project: openstack/airship-shipyard Branch: refs/heads/master
-rw-r--r--charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl23
-rw-r--r--charts/shipyard/templates/configmap-airflow-bin.yaml2
-rw-r--r--charts/shipyard/templates/deployment-airflow-scheduler.yaml44
-rw-r--r--charts/shipyard/templates/deployment-airflow-web.yaml44
-rw-r--r--charts/shipyard/templates/statefulset-airflow-worker.yaml69
-rw-r--r--charts/shipyard/values.yaml6
-rw-r--r--images/airflow/Dockerfile16
7 files changed, 15 insertions, 189 deletions
diff --git a/charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl b/charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl
deleted file mode 100644
index ef72f5c..0000000
--- a/charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl
+++ /dev/null
@@ -1,23 +0,0 @@
1#!/bin/bash
2
3{{/*
4Copyright 2017 The Openstack-Helm Authors.
5Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
6
7Licensed under the Apache License, Version 2.0 (the "License");
8you may not use this file except in compliance with the License.
9You may obtain a copy of the License at
10
11 http://www.apache.org/licenses/LICENSE-2.0
12
13Unless required by applicable law or agreed to in writing, software
14distributed under the License is distributed on an "AS IS" BASIS,
15WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16See the License for the specific language governing permissions and
17limitations under the License.
18*/}}
19
20set -ex
21
22cp -vaR /home/shipyard/shipyard/shipyard_airflow/dags/*.py /tmp/airflow/dags/
23cp -vaR /home/shipyard/shipyard/shipyard_airflow/plugins/*.py /tmp/airflow/plugins/
diff --git a/charts/shipyard/templates/configmap-airflow-bin.yaml b/charts/shipyard/templates/configmap-airflow-bin.yaml
index 12ab0f1..1850022 100644
--- a/charts/shipyard/templates/configmap-airflow-bin.yaml
+++ b/charts/shipyard/templates/configmap-airflow-bin.yaml
@@ -27,6 +27,4 @@ data:
27{{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} 27{{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
28 airflow-db-sync.sh: |+ 28 airflow-db-sync.sh: |+
29{{ tuple "bin/_airflow-db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} 29{{ tuple "bin/_airflow-db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
30 airflow-shipyard-init.sh: |+
31{{ tuple "bin/_airflow-shipyard-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
32{{- end }} 30{{- end }}
diff --git a/charts/shipyard/templates/deployment-airflow-scheduler.yaml b/charts/shipyard/templates/deployment-airflow-scheduler.yaml
index 8c6c597..ff278aa 100644
--- a/charts/shipyard/templates/deployment-airflow-scheduler.yaml
+++ b/charts/shipyard/templates/deployment-airflow-scheduler.yaml
@@ -53,23 +53,6 @@ spec:
53 terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} 53 terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }}
54 initContainers: 54 initContainers:
55{{ tuple $envAll "airflow_server" $mounts_airflow_scheduler_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} 55{{ tuple $envAll "airflow_server" $mounts_airflow_scheduler_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
56{{- if .Values.prod_environment }}
57 - name: airflow-shipyard-init
58 image: {{ .Values.images.tags.shipyard }}
59 imagePullPolicy: {{ .Values.images.pull_policy }}
60{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
61 command:
62 - /tmp/airflow-shipyard-init.sh
63 volumeMounts:
64 - name: airflow-bin
65 mountPath: /tmp/airflow-shipyard-init.sh
66 subPath: airflow-shipyard-init.sh
67 readOnly: true
68 - name: pod-shipyard-share-airflow-dags
69 mountPath: /tmp/airflow/dags/
70 - name: pod-shipyard-share-airflow-plugins
71 mountPath: /tmp/airflow/plugins/
72{{- end }}
73 containers: 56 containers:
74 - name: airflow-scheduler 57 - name: airflow-scheduler
75 image: {{ .Values.images.tags.airflow }} 58 image: {{ .Values.images.tags.airflow }}
@@ -88,21 +71,10 @@ spec:
88 mountPath: {{ .Values.conf.airflow_config_file.path }} 71 mountPath: {{ .Values.conf.airflow_config_file.path }}
89 subPath: airflow.cfg 72 subPath: airflow.cfg
90 readOnly: true 73 readOnly: true
91{{- if .Values.prod_environment }}
92 - name: shipyard-etc 74 - name: shipyard-etc
93 mountPath: /usr/local/airflow/plugins/shipyard.conf 75 mountPath: /usr/local/airflow/plugins/shipyard.conf
94 subPath: shipyard.conf 76 subPath: shipyard.conf
95 readOnly: true 77 readOnly: true
96 - name: pod-shipyard-share-airflow-dags
97 mountPath: {{ .Values.conf.airflow.core.dags_folder }}
98 - name: pod-shipyard-share-airflow-plugins
99 mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
100{{ else }}
101 - name: airflow-dags
102 mountPath: {{ .Values.conf.airflow.core.dags_folder }}
103 - name: airflow-plugins
104 mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
105{{- end }}
106 - name: airflow-logs 78 - name: airflow-logs
107 mountPath: {{ .Values.conf.airflow.core.base_log_folder }} 79 mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
108{{ if $mounts_airflow_scheduler.volumeMounts }}{{ toYaml $mounts_airflow_scheduler.volumeMounts | indent 12 }}{{ end }} 80{{ if $mounts_airflow_scheduler.volumeMounts }}{{ toYaml $mounts_airflow_scheduler.volumeMounts | indent 12 }}{{ end }}
@@ -111,7 +83,6 @@ spec:
111 configMap: 83 configMap:
112 name: airflow-etc 84 name: airflow-etc
113 defaultMode: 0444 85 defaultMode: 0444
114{{- if .Values.prod_environment }}
115 - name: shipyard-etc 86 - name: shipyard-etc
116 configMap: 87 configMap:
117 name: shipyard-etc 88 name: shipyard-etc
@@ -120,22 +91,7 @@ spec:
120 configMap: 91 configMap:
121 name: airflow-bin 92 name: airflow-bin
122 defaultMode: 0555 93 defaultMode: 0555
123 - name: pod-shipyard-share-airflow-dags
124 emptyDir: {}
125 - name: pod-shipyard-share-airflow-plugins
126 emptyDir: {}
127 - name: airflow-logs 94 - name: airflow-logs
128 emptyDir: {} 95 emptyDir: {}
129{{ else }}
130 - name: airflow-dags
131 hostPath:
132 path: {{ .Values.pod.mounts.dag_path }}
133 - name: airflow-plugins
134 hostPath:
135 path: {{ .Values.pod.mounts.plugin_path }}
136 - name: airflow-logs
137 hostPath:
138 path: {{ .Values.pod.mounts.log_path }}
139{{- end }}
140{{ if $mounts_airflow_scheduler.volumes }}{{ toYaml $mounts_airflow_scheduler.volumes | indent 8 }}{{ end }} 96{{ if $mounts_airflow_scheduler.volumes }}{{ toYaml $mounts_airflow_scheduler.volumes | indent 8 }}{{ end }}
141{{- end }} 97{{- end }}
diff --git a/charts/shipyard/templates/deployment-airflow-web.yaml b/charts/shipyard/templates/deployment-airflow-web.yaml
index faaecb6..648f556 100644
--- a/charts/shipyard/templates/deployment-airflow-web.yaml
+++ b/charts/shipyard/templates/deployment-airflow-web.yaml
@@ -44,23 +44,6 @@ spec:
44 terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} 44 terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }}
45 initContainers: 45 initContainers:
46{{ tuple $envAll "airflow_server" $mounts_airflow_web_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} 46{{ tuple $envAll "airflow_server" $mounts_airflow_web_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
47{{- if .Values.prod_environment }}
48 - name: airflow-shipyard-init
49 image: {{ .Values.images.tags.shipyard }}
50 imagePullPolicy: {{ .Values.images.pull_policy }}
51{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
52 command:
53 - /tmp/airflow-shipyard-init.sh
54 volumeMounts:
55 - name: airflow-bin
56 mountPath: /tmp/airflow-shipyard-init.sh
57 subPath: airflow-shipyard-init.sh
58 readOnly: true
59 - name: pod-shipyard-share-airflow-dags
60 mountPath: /tmp/airflow/dags/
61 - name: pod-shipyard-share-airflow-plugins
62 mountPath: /tmp/airflow/plugins/
63{{- end }}
64 containers: 47 containers:
65 - name: airflow-web 48 - name: airflow-web
66 image: {{ .Values.images.tags.airflow }} 49 image: {{ .Values.images.tags.airflow }}
@@ -83,21 +66,10 @@ spec:
83 mountPath: {{ .Values.conf.airflow_config_file.path }} 66 mountPath: {{ .Values.conf.airflow_config_file.path }}
84 subPath: airflow.cfg 67 subPath: airflow.cfg
85 readOnly: true 68 readOnly: true
86{{- if .Values.prod_environment }}
87 - name: shipyard-etc 69 - name: shipyard-etc
88 mountPath: /usr/local/airflow/plugins/shipyard.conf 70 mountPath: /usr/local/airflow/plugins/shipyard.conf
89 subPath: shipyard.conf 71 subPath: shipyard.conf
90 readOnly: true 72 readOnly: true
91 - name: pod-shipyard-share-airflow-dags
92 mountPath: {{ .Values.conf.airflow.core.dags_folder }}
93 - name: pod-shipyard-share-airflow-plugins
94 mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
95{{ else }}
96 - name: airflow-dags
97 mountPath: {{ .Values.conf.airflow.core.dags_folder }}
98 - name: airflow-plugins
99 mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
100{{- end }}
101 - name: airflow-logs 73 - name: airflow-logs
102 mountPath: {{ .Values.conf.airflow.core.base_log_folder }} 74 mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
103{{ if $mounts_airflow_web.volumeMounts }}{{ toYaml $mounts_airflow_web.volumeMounts | indent 12 }}{{ end }} 75{{ if $mounts_airflow_web.volumeMounts }}{{ toYaml $mounts_airflow_web.volumeMounts | indent 12 }}{{ end }}
@@ -106,7 +78,6 @@ spec:
106 configMap: 78 configMap:
107 name: airflow-etc 79 name: airflow-etc
108 defaultMode: 0444 80 defaultMode: 0444
109{{- if .Values.prod_environment }}
110 - name: shipyard-etc 81 - name: shipyard-etc
111 configMap: 82 configMap:
112 name: shipyard-etc 83 name: shipyard-etc
@@ -115,22 +86,7 @@ spec:
115 configMap: 86 configMap:
116 name: airflow-bin 87 name: airflow-bin
117 defaultMode: 0555 88 defaultMode: 0555
118 - name: pod-shipyard-share-airflow-dags
119 emptyDir: {}
120 - name: pod-shipyard-share-airflow-plugins
121 emptyDir: {}
122 - name: airflow-logs 89 - name: airflow-logs
123 emptyDir: {} 90 emptyDir: {}
124{{ else }}
125 - name: airflow-dags
126 hostPath:
127 path: {{ .Values.pod.mounts.dag_path }}
128 - name: airflow-plugins
129 hostPath:
130 path: {{ .Values.pod.mounts.plugin_path }}
131 - name: airflow-logs
132 hostPath:
133 path: {{ .Values.pod.mounts.log_path }}
134{{- end }}
135{{ if $mounts_airflow_web.volumes }}{{ toYaml $mounts_airflow_web.volumes | indent 8 }}{{ end }} 91{{ if $mounts_airflow_web.volumes }}{{ toYaml $mounts_airflow_web.volumes | indent 8 }}{{ end }}
136{{- end }} 92{{- end }}
diff --git a/charts/shipyard/templates/statefulset-airflow-worker.yaml b/charts/shipyard/templates/statefulset-airflow-worker.yaml
index a53473d..c71b63e 100644
--- a/charts/shipyard/templates/statefulset-airflow-worker.yaml
+++ b/charts/shipyard/templates/statefulset-airflow-worker.yaml
@@ -79,22 +79,6 @@ spec:
79 {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }} 79 {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }}
80 initContainers: 80 initContainers:
81{{ tuple $envAll "airflow_server" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} 81{{ tuple $envAll "airflow_server" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
82{{- if .Values.prod_environment }}
83 - name: airflow-shipyard-init
84 image: {{ .Values.images.tags.shipyard }}
85 imagePullPolicy: {{ .Values.images.pull_policy }}
86{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
87 command:
88 - /tmp/airflow-shipyard-init.sh
89 volumeMounts:
90 - name: airflow-bin
91 mountPath: /tmp/airflow-shipyard-init.sh
92 subPath: airflow-shipyard-init.sh
93 readOnly: true
94 - name: pod-shipyard-share-airflow-dags
95 mountPath: /tmp/airflow/dags/
96 - name: pod-shipyard-share-airflow-plugins
97 mountPath: /tmp/airflow/plugins/
98 - name: worker-perms 82 - name: worker-perms
99 image: {{ .Values.images.tags.airflow }} 83 image: {{ .Values.images.tags.airflow }}
100 imagePullPolicy: {{ .Values.images.pull_policy }} 84 imagePullPolicy: {{ .Values.images.pull_policy }}
@@ -109,7 +93,6 @@ spec:
109 volumeMounts: 93 volumeMounts:
110 - name: airflow-logs 94 - name: airflow-logs
111 mountPath: {{ .Values.conf.airflow.core.base_log_folder }} 95 mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
112{{- end }}
113 containers: 96 containers:
114 - name: airflow-scheduler 97 - name: airflow-scheduler
115 image: {{ .Values.images.tags.airflow }} 98 image: {{ .Values.images.tags.airflow }}
@@ -128,21 +111,10 @@ spec:
128 mountPath: {{ .Values.conf.airflow_config_file.path }} 111 mountPath: {{ .Values.conf.airflow_config_file.path }}
129 subPath: airflow.cfg 112 subPath: airflow.cfg
130 readOnly: true 113 readOnly: true
131{{- if .Values.prod_environment }}
132 - name: shipyard-etc 114 - name: shipyard-etc
133 mountPath: /usr/local/airflow/plugins/shipyard.conf 115 mountPath: /usr/local/airflow/plugins/shipyard.conf
134 subPath: shipyard.conf 116 subPath: shipyard.conf
135 readOnly: true 117 readOnly: true
136 - name: pod-shipyard-share-airflow-dags
137 mountPath: {{ .Values.conf.airflow.core.dags_folder }}
138 - name: pod-shipyard-share-airflow-plugins
139 mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
140{{ else }}
141 - name: airflow-dags
142 mountPath: {{ .Values.conf.airflow.core.dags_folder }}
143 - name: airflow-plugins
144 mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
145{{- end }}
146 - name: airflow-logs 118 - name: airflow-logs
147 mountPath: {{ .Values.conf.airflow.core.base_log_folder }} 119 mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
148 - name: airflow-worker 120 - name: airflow-worker
@@ -166,30 +138,12 @@ spec:
166 mountPath: {{ .Values.conf.airflow_config_file.path }} 138 mountPath: {{ .Values.conf.airflow_config_file.path }}
167 subPath: airflow.cfg 139 subPath: airflow.cfg
168 readOnly: true 140 readOnly: true
169{{- if .Values.prod_environment }}
170 - name: shipyard-etc 141 - name: shipyard-etc
171 mountPath: /usr/local/airflow/plugins/shipyard.conf 142 mountPath: /usr/local/airflow/plugins/shipyard.conf
172 subPath: shipyard.conf 143 subPath: shipyard.conf
173 readOnly: true 144 readOnly: true
174 - name: pod-shipyard-share-airflow-dags
175 mountPath: {{ .Values.conf.airflow.core.dags_folder }}
176 - name: pod-shipyard-share-airflow-plugins
177 mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
178{{ else }}
179 - name: airflow-dags
180 mountPath: {{ .Values.conf.airflow.core.dags_folder }}
181 - name: airflow-plugins
182 mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
183 - name: docker
184 mountPath: /var/run
185 readOnly: false
186 - name: pod-var-lib-docker
187 mountPath: /var/lib/docker
188 readOnly: false
189{{- end }}
190 - name: airflow-logs 145 - name: airflow-logs
191 mountPath: {{ .Values.conf.airflow.core.base_log_folder }} 146 mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
192{{- if .Values.prod_environment }}
193 - name: airflow-logrotate 147 - name: airflow-logrotate
194 image: {{ .Values.images.tags.airflow }} 148 image: {{ .Values.images.tags.airflow }}
195 imagePullPolicy: {{ .Values.images.pull_policy }} 149 imagePullPolicy: {{ .Values.images.pull_policy }}
@@ -214,13 +168,11 @@ spec:
214 volumeMounts: 168 volumeMounts:
215 - name: airflow-logs 169 - name: airflow-logs
216 mountPath: {{ .Values.conf.airflow.core.base_log_folder }} 170 mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
217{{- end }}
218 volumes: 171 volumes:
219 - name: airflow-etc 172 - name: airflow-etc
220 configMap: 173 configMap:
221 name: airflow-etc 174 name: airflow-etc
222 defaultMode: 0444 175 defaultMode: 0444
223{{- if .Values.prod_environment }}
224 - name: shipyard-etc 176 - name: shipyard-etc
225 configMap: 177 configMap:
226 name: shipyard-etc 178 name: shipyard-etc
@@ -229,10 +181,6 @@ spec:
229 configMap: 181 configMap:
230 name: airflow-bin 182 name: airflow-bin
231 defaultMode: 0555 183 defaultMode: 0555
232 - name: pod-shipyard-share-airflow-dags
233 emptyDir: {}
234 - name: pod-shipyard-share-airflow-plugins
235 emptyDir: {}
236 volumeClaimTemplates: 184 volumeClaimTemplates:
237 - metadata: 185 - metadata:
238 name: airflow-logs 186 name: airflow-logs
@@ -242,21 +190,4 @@ spec:
242 requests: 190 requests:
243 storage: {{ .Values.volume_worker.size }} 191 storage: {{ .Values.volume_worker.size }}
244 storageClassName: {{ .Values.volume_worker.class_name }} 192 storageClassName: {{ .Values.volume_worker.class_name }}
245{{ else }}
246 - name: airflow-dags
247 hostPath:
248 path: {{ .Values.pod.mounts.dag_path }}
249 - name: airflow-plugins
250 hostPath:
251 path: {{ .Values.pod.mounts.plugin_path }}
252 - name: airflow-logs
253 hostPath:
254 path: {{ .Values.pod.mounts.log_path }}
255 - name: docker
256 hostPath:
257 path: /var/run
258 - name: pod-var-lib-docker
259 hostPath:
260 path: /var/lib/docker
261{{- end }}
262{{- end }} 193{{- end }}
diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml
index 969ad4b..998a429 100644
--- a/charts/shipyard/values.yaml
+++ b/charts/shipyard/values.yaml
@@ -15,9 +15,6 @@
15 15
16# This file provides defaults for shipyard and airflow 16# This file provides defaults for shipyard and airflow
17 17
18# Indicate whether it is production or development environment
19prod_environment: true
20
21labels: 18labels:
22 job: 19 job:
23 node_selector_key: ucp-control-plane 20 node_selector_key: ucp-control-plane
@@ -701,9 +698,6 @@ conf:
701 # End of Airflow config options 698 # End of Airflow config options
702pod: 699pod:
703 mounts: 700 mounts:
704 dag_path: /home/ubuntu/workbench/dags
705 plugin_path: /home/ubuntu/workbench/plugins
706 log_path: /home/ubuntu/workbench/logs
707 airflow_scheduler: 701 airflow_scheduler:
708 # TODO: This is only used if the standalone scheduler is enabled. 702 # TODO: This is only used if the standalone scheduler is enabled.
709 airflow_scheduler: 703 airflow_scheduler:
diff --git a/images/airflow/Dockerfile b/images/airflow/Dockerfile
index 8652f15..6b13096 100644
--- a/images/airflow/Dockerfile
+++ b/images/airflow/Dockerfile
@@ -104,7 +104,7 @@ ENV PBR_VERSION 0.1a1
104 104
105# Shipyard 105# Shipyard
106# 106#
107# Shipyard provides core functionality used by the airflow plugins/operators 107# Shipyard provides core functionality used by the Airflow plugins/operators
108# Since Shipyard and Airflow are built together as images, this should prevent 108# Since Shipyard and Airflow are built together as images, this should prevent
109# stale or out-of-date code between these parts. 109# stale or out-of-date code between these parts.
110# Shipyard requirements, source and installation 110# Shipyard requirements, source and installation
@@ -115,6 +115,20 @@ COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
115RUN cd /tmp/shipyard \ 115RUN cd /tmp/shipyard \
116 && python3 setup.py install 116 && python3 setup.py install
117 117
118# Note: The value for the dags and plugins directories that are sourced
119# from the values.yaml of the Shipyard Helm chart need to align with these
120# directories. If they do not, airflow will not find the intended dags and
121# plugins.
122#
123# Note: In the case of building images using the provided Makefile, a test is
124# run against the built-in dags provided with Airflow. Since there is no Helm
125# chart to reconfigure the airflow.cfg with these directories, these dags and
126# plugins are not known to Airflow during the image test.
127#
128# Copy the plugins and dags that will be used by this Airflow image:
129COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/
130COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/
131
118# Set work directory 132# Set work directory
119USER airflow 133USER airflow
120WORKDIR ${AIRFLOW_HOME} 134WORKDIR ${AIRFLOW_HOME}