summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2018-12-28 08:39:39 +0000
committerGerrit Code Review <review@openstack.org>2018-12-28 08:39:39 +0000
commit4dd6b484d11e86ad51da733841b9ef137421d461 (patch)
tree3afd878fcd57780d82355e87ec4a0bc4f3124b24
parentc1b12b9a9e37756749743e1e3404c7b15d853fdc (diff)
parent2134a878755722bab716e9aace53f9e98720b691 (diff)
Merge "Setup mirrored queues for celery in rabbit"
-rw-r--r--charts/shipyard/templates/configmap-airflow-bin.yaml2
-rw-r--r--charts/shipyard/templates/job-rabbit-init.yaml20
-rw-r--r--charts/shipyard/templates/secret-rabbitmq.yaml28
-rw-r--r--charts/shipyard/templates/secret-user-rabbitmq.yaml28
-rw-r--r--charts/shipyard/values.yaml36
-rw-r--r--tools/gate/playbooks/zuul-linter.yaml2
6 files changed, 114 insertions, 2 deletions
diff --git a/charts/shipyard/templates/configmap-airflow-bin.yaml b/charts/shipyard/templates/configmap-airflow-bin.yaml
index c19562b..12ab0f1 100644
--- a/charts/shipyard/templates/configmap-airflow-bin.yaml
+++ b/charts/shipyard/templates/configmap-airflow-bin.yaml
@@ -21,6 +21,8 @@ kind: ConfigMap
21metadata: 21metadata:
22 name: airflow-bin 22 name: airflow-bin
23data: 23data:
24 rabbit-init.sh: |+
25{{- include "helm-toolkit.scripts.rabbit_init" . | indent 4 }}
24 airflow-db-init.sh: |+ 26 airflow-db-init.sh: |+
25{{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} 27{{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
26 airflow-db-sync.sh: |+ 28 airflow-db-sync.sh: |+
diff --git a/charts/shipyard/templates/job-rabbit-init.yaml b/charts/shipyard/templates/job-rabbit-init.yaml
new file mode 100644
index 0000000..39aa527
--- /dev/null
+++ b/charts/shipyard/templates/job-rabbit-init.yaml
@@ -0,0 +1,20 @@
1{{/*
2Copyright 2018 The Openstack-Helm Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/}}
16
17{{- if .Values.manifests.job_rabbit_init }}
18{{- $rmqJob := dict "envAll" . "serviceName" "airflow" -}}
19{{ $rmqJob | include "helm-toolkit.manifests.job_rabbit_init" }}
20{{- end }} \ No newline at end of file
diff --git a/charts/shipyard/templates/secret-rabbitmq.yaml b/charts/shipyard/templates/secret-rabbitmq.yaml
new file mode 100644
index 0000000..235ae22
--- /dev/null
+++ b/charts/shipyard/templates/secret-rabbitmq.yaml
@@ -0,0 +1,28 @@
1{{/*
2Copyright 2018 The Openstack-Helm Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/}}
16
17{{- if .Values.manifests.secret_rabbitmq }}
18{{- $envAll := . }}
19{{- $secretName := index $envAll.Values.secrets.oslo_messaging "admin" }}
20---
21apiVersion: v1
22kind: Secret
23metadata:
24 name: {{ $secretName }}
25type: Opaque
26data:
27 RABBITMQ_CONNECTION: {{ tuple "oslo_messaging" "internal" "admin" "http" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc }}
28{{- end }}
diff --git a/charts/shipyard/templates/secret-user-rabbitmq.yaml b/charts/shipyard/templates/secret-user-rabbitmq.yaml
new file mode 100644
index 0000000..c69795d
--- /dev/null
+++ b/charts/shipyard/templates/secret-user-rabbitmq.yaml
@@ -0,0 +1,28 @@
1{{/*
2Copyright 2018 The Openstack-Helm Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/}}
16
17{{- if .Values.manifests.secret_rabbitmq }}
18{{- $envAll := . }}
19{{- $secretName := index $envAll.Values.secrets.oslo_messaging "airflow" }}
20---
21apiVersion: v1
22kind: Secret
23metadata:
24 name: {{ $secretName }}
25type: Opaque
26data:
27 RABBITMQ_CONNECTION: {{ tuple "oslo_messaging" "internal" "user" "amqp" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc }}
28{{- end }} \ No newline at end of file
diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml
index 6688bd8..3c88bd8 100644
--- a/charts/shipyard/values.yaml
+++ b/charts/shipyard/values.yaml
@@ -37,6 +37,7 @@ images:
37 shipyard_db_init: docker.io/postgres:9.5 37 shipyard_db_init: docker.io/postgres:9.5
38 shipyard_db_sync: quay.io/airshipit/shipyard:latest 38 shipyard_db_sync: quay.io/airshipit/shipyard:latest
39 airflow_db_init: docker.io/postgres:9.5 39 airflow_db_init: docker.io/postgres:9.5
40 rabbit_init: docker.io/rabbitmq:3.7-management
40 airflow_db_sync: quay.io/airshipit/airflow:latest 41 airflow_db_sync: quay.io/airshipit/airflow:latest
41 ks_user: docker.io/openstackhelm/heat:ocata 42 ks_user: docker.io/openstackhelm/heat:ocata
42 ks_service: docker.io/openstackhelm/heat:ocata 43 ks_service: docker.io/openstackhelm/heat:ocata
@@ -105,6 +106,10 @@ dependencies:
105 services: 106 services:
106 - service: postgresql_airflow_db 107 - service: postgresql_airflow_db
107 endpoint: internal 108 endpoint: internal
109 rabbit_init:
110 services:
111 - service: oslo_messaging
112 endpoint: internal
108 airflow_db_sync: 113 airflow_db_sync:
109 jobs: 114 jobs:
110 - airflow-db-init 115 - airflow-db-init
@@ -143,6 +148,7 @@ dependencies:
143 endpoint: internal 148 endpoint: internal
144 airflow_server: 149 airflow_server:
145 jobs: 150 jobs:
151 - airflow-rabbit-init
146 - airflow-db-init 152 - airflow-db-init
147 - airflow-db-sync 153 - airflow-db-sync
148 services: 154 services:
@@ -316,17 +322,22 @@ endpoints:
316 oslo_messaging: 322 oslo_messaging:
317 auth: 323 auth:
318 user: 324 user:
325 username: airflow
326 password: password
327 admin:
319 username: rabbitmq 328 username: rabbitmq
320 password: password 329 password: password
321 hosts: 330 hosts:
322 default: rabbitmq 331 default: rabbitmq
323 host_fqdn_override: 332 host_fqdn_override:
324 default: null 333 default: null
325 path: / 334 path: /airflow
326 scheme: amqp 335 scheme: amqp
327 port: 336 port:
328 amqp: 337 amqp:
329 default: 5672 338 default: 5672
339 http:
340 default: 15672
330 oslo_cache: 341 oslo_cache:
331 hosts: 342 hosts:
332 default: memcached 343 default: memcached
@@ -340,6 +351,9 @@ secrets:
340 identity: 351 identity:
341 admin: shipyard-keystone-admin 352 admin: shipyard-keystone-admin
342 shipyard: shipyard-keystone-user 353 shipyard: shipyard-keystone-user
354 oslo_messaging:
355 admin: airflow-rabbitmq-admin
356 airflow: airflow-rabbitmq-user
343 postgresql_shipyard_db: 357 postgresql_shipyard_db:
344 admin: shipyard-db-admin 358 admin: shipyard-db-admin
345 user: shipyard-db-user 359 user: shipyard-db-user
@@ -381,6 +395,17 @@ conf:
381 workflow_orchestrator:action_update_software: rule:admin_create 395 workflow_orchestrator:action_update_software: rule:admin_create
382 workflow_orchestrator:action_redeploy_server: rule:admin_create 396 workflow_orchestrator:action_redeploy_server: rule:admin_create
383 workflow_orchestrator:action_relabel_nodes: rule:admin_create 397 workflow_orchestrator:action_relabel_nodes: rule:admin_create
398 rabbitmq:
399 # adding rmq policy to mirror messages from celery queues
400 policies:
401 - vhost: "/"
402 name: "ha_celery"
403 definition:
404 ha-mode: "all"
405 ha-sync-mode: "automatic"
406 priority: 0
407 apply-to: all
408 pattern: 'celery.*'
384 paste: 409 paste:
385 app:shipyard-api: 410 app:shipyard-api:
386 paste.app_factory: shipyard_airflow.shipyard_api:paste_start_shipyard 411 paste.app_factory: shipyard_airflow.shipyard_api:paste_start_shipyard
@@ -785,6 +810,13 @@ pod:
785 memory: "128Mi" 810 memory: "128Mi"
786 cpu: "100m" 811 cpu: "100m"
787 jobs: 812 jobs:
813 rabbit_init:
814 limits:
815 memory: "128Mi"
816 cpu: "500m"
817 requests:
818 memory: "128Mi"
819 cpu: "500m"
788 airflow_db_init: 820 airflow_db_init:
789 limits: 821 limits:
790 memory: "128Mi" 822 memory: "128Mi"
@@ -864,6 +896,7 @@ manifests:
864 ingress_shipyard_api: true 896 ingress_shipyard_api: true
865 job_shipyard_db_init: true 897 job_shipyard_db_init: true
866 job_shipyard_db_sync: true 898 job_shipyard_db_sync: true
899 job_rabbit_init: true
867 job_airflow_db_init: true 900 job_airflow_db_init: true
868 job_airflow_db_sync: true 901 job_airflow_db_sync: true
869 job_ks_endpoints: true 902 job_ks_endpoints: true
@@ -873,6 +906,7 @@ manifests:
873 secret_shipyard_db: true 906 secret_shipyard_db: true
874 secret_ingress_tls: true 907 secret_ingress_tls: true
875 secret_keystone: true 908 secret_keystone: true
909 secret_rabbitmq: true
876 service_airflow_ingress: true 910 service_airflow_ingress: true
877 service_airflow_flower: true 911 service_airflow_flower: true
878 service_shipyard: true 912 service_shipyard: true
diff --git a/tools/gate/playbooks/zuul-linter.yaml b/tools/gate/playbooks/zuul-linter.yaml
index 7b6fb37..64a8926 100644
--- a/tools/gate/playbooks/zuul-linter.yaml
+++ b/tools/gate/playbooks/zuul-linter.yaml
@@ -15,6 +15,6 @@
15- hosts: primary 15- hosts: primary
16 tasks: 16 tasks:
17 - name: Execute a Whitespace Linter check 17 - name: Execute a Whitespace Linter check
18 command: find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -type f -exec egrep -l " +$" {} \; 18 command: find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -not -name "*.png" -type f -exec egrep -l " +$" {} \;
19 register: result 19 register: result
20 failed_when: result.stdout != "" \ No newline at end of file 20 failed_when: result.stdout != "" \ No newline at end of file