summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorVladyslav Drok <vdrok@mirantis.com>2018-11-06 15:07:58 -0800
committerVladyslav Drok <vdrok@mirantis.com>2018-12-17 19:37:28 +0000
commit2134a878755722bab716e9aace53f9e98720b691 (patch)
tree5b28953f8cc9211158e319064241c72a59ac482d
parentbae15294c534cf321f5c7ca37592dfa74c4ad7c2 (diff)
Setup mirrored queues for celery in rabbit
This change starts setting up mirrored queues by default. If there is only one rabbitmq pod present this will still work, though will cause some performance overhead. Depends-On: https://review.openstack.org/617812 Depends-On: https://review.openstack.org/617817 Change-Id: I8982aed699185f9b7fb4962e108eb76377643f25
Notes
Notes (review): Code-Review+2: Scott Hussey <sthussey@att.com> Code-Review+1: Chris Wedgwood <cw@f00f.org> Code-Review+1: Evgeniy L <eli@mirantis.com> Code-Review+2: Pete Birley <petebirley@gmail.com> Workflow+1: Pete Birley <petebirley@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Fri, 28 Dec 2018 08:39:39 +0000 Reviewed-on: https://review.openstack.org/614610 Project: openstack/airship-shipyard Branch: refs/heads/master
-rw-r--r--charts/shipyard/templates/configmap-airflow-bin.yaml2
-rw-r--r--charts/shipyard/templates/job-rabbit-init.yaml20
-rw-r--r--charts/shipyard/templates/secret-rabbitmq.yaml28
-rw-r--r--charts/shipyard/templates/secret-user-rabbitmq.yaml28
-rw-r--r--charts/shipyard/values.yaml36
-rw-r--r--tools/gate/playbooks/zuul-linter.yaml2
6 files changed, 114 insertions, 2 deletions
diff --git a/charts/shipyard/templates/configmap-airflow-bin.yaml b/charts/shipyard/templates/configmap-airflow-bin.yaml
index c19562b..12ab0f1 100644
--- a/charts/shipyard/templates/configmap-airflow-bin.yaml
+++ b/charts/shipyard/templates/configmap-airflow-bin.yaml
@@ -21,6 +21,8 @@ kind: ConfigMap
21metadata: 21metadata:
22 name: airflow-bin 22 name: airflow-bin
23data: 23data:
24 rabbit-init.sh: |+
25{{- include "helm-toolkit.scripts.rabbit_init" . | indent 4 }}
24 airflow-db-init.sh: |+ 26 airflow-db-init.sh: |+
25{{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} 27{{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
26 airflow-db-sync.sh: |+ 28 airflow-db-sync.sh: |+
diff --git a/charts/shipyard/templates/job-rabbit-init.yaml b/charts/shipyard/templates/job-rabbit-init.yaml
new file mode 100644
index 0000000..39aa527
--- /dev/null
+++ b/charts/shipyard/templates/job-rabbit-init.yaml
@@ -0,0 +1,20 @@
1{{/*
2Copyright 2018 The Openstack-Helm Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/}}
16
17{{- if .Values.manifests.job_rabbit_init }}
18{{- $rmqJob := dict "envAll" . "serviceName" "airflow" -}}
19{{ $rmqJob | include "helm-toolkit.manifests.job_rabbit_init" }}
20{{- end }} \ No newline at end of file
diff --git a/charts/shipyard/templates/secret-rabbitmq.yaml b/charts/shipyard/templates/secret-rabbitmq.yaml
new file mode 100644
index 0000000..235ae22
--- /dev/null
+++ b/charts/shipyard/templates/secret-rabbitmq.yaml
@@ -0,0 +1,28 @@
1{{/*
2Copyright 2018 The Openstack-Helm Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/}}
16
17{{- if .Values.manifests.secret_rabbitmq }}
18{{- $envAll := . }}
19{{- $secretName := index $envAll.Values.secrets.oslo_messaging "admin" }}
20---
21apiVersion: v1
22kind: Secret
23metadata:
24 name: {{ $secretName }}
25type: Opaque
26data:
27 RABBITMQ_CONNECTION: {{ tuple "oslo_messaging" "internal" "admin" "http" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc }}
28{{- end }}
diff --git a/charts/shipyard/templates/secret-user-rabbitmq.yaml b/charts/shipyard/templates/secret-user-rabbitmq.yaml
new file mode 100644
index 0000000..c69795d
--- /dev/null
+++ b/charts/shipyard/templates/secret-user-rabbitmq.yaml
@@ -0,0 +1,28 @@
1{{/*
2Copyright 2018 The Openstack-Helm Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/}}
16
17{{- if .Values.manifests.secret_rabbitmq }}
18{{- $envAll := . }}
19{{- $secretName := index $envAll.Values.secrets.oslo_messaging "airflow" }}
20---
21apiVersion: v1
22kind: Secret
23metadata:
24 name: {{ $secretName }}
25type: Opaque
26data:
27 RABBITMQ_CONNECTION: {{ tuple "oslo_messaging" "internal" "user" "amqp" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc }}
28{{- end }} \ No newline at end of file
diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml
index 32b51a4..a6ee526 100644
--- a/charts/shipyard/values.yaml
+++ b/charts/shipyard/values.yaml
@@ -37,6 +37,7 @@ images:
37 shipyard_db_init: docker.io/postgres:9.5 37 shipyard_db_init: docker.io/postgres:9.5
38 shipyard_db_sync: quay.io/airshipit/shipyard:latest 38 shipyard_db_sync: quay.io/airshipit/shipyard:latest
39 airflow_db_init: docker.io/postgres:9.5 39 airflow_db_init: docker.io/postgres:9.5
40 rabbit_init: docker.io/rabbitmq:3.7-management
40 airflow_db_sync: quay.io/airshipit/airflow:latest 41 airflow_db_sync: quay.io/airshipit/airflow:latest
41 ks_user: docker.io/openstackhelm/heat:ocata 42 ks_user: docker.io/openstackhelm/heat:ocata
42 ks_service: docker.io/openstackhelm/heat:ocata 43 ks_service: docker.io/openstackhelm/heat:ocata
@@ -105,6 +106,10 @@ dependencies:
105 services: 106 services:
106 - service: postgresql_airflow_db 107 - service: postgresql_airflow_db
107 endpoint: internal 108 endpoint: internal
109 rabbit_init:
110 services:
111 - service: oslo_messaging
112 endpoint: internal
108 airflow_db_sync: 113 airflow_db_sync:
109 jobs: 114 jobs:
110 - airflow-db-init 115 - airflow-db-init
@@ -143,6 +148,7 @@ dependencies:
143 endpoint: internal 148 endpoint: internal
144 airflow_server: 149 airflow_server:
145 jobs: 150 jobs:
151 - airflow-rabbit-init
146 - airflow-db-init 152 - airflow-db-init
147 - airflow-db-sync 153 - airflow-db-sync
148 services: 154 services:
@@ -316,17 +322,22 @@ endpoints:
316 oslo_messaging: 322 oslo_messaging:
317 auth: 323 auth:
318 user: 324 user:
325 username: airflow
326 password: password
327 admin:
319 username: rabbitmq 328 username: rabbitmq
320 password: password 329 password: password
321 hosts: 330 hosts:
322 default: rabbitmq 331 default: rabbitmq
323 host_fqdn_override: 332 host_fqdn_override:
324 default: null 333 default: null
325 path: / 334 path: /airflow
326 scheme: amqp 335 scheme: amqp
327 port: 336 port:
328 amqp: 337 amqp:
329 default: 5672 338 default: 5672
339 http:
340 default: 15672
330 oslo_cache: 341 oslo_cache:
331 hosts: 342 hosts:
332 default: memcached 343 default: memcached
@@ -340,6 +351,9 @@ secrets:
340 identity: 351 identity:
341 admin: shipyard-keystone-admin 352 admin: shipyard-keystone-admin
342 shipyard: shipyard-keystone-user 353 shipyard: shipyard-keystone-user
354 oslo_messaging:
355 admin: airflow-rabbitmq-admin
356 airflow: airflow-rabbitmq-user
343 postgresql_shipyard_db: 357 postgresql_shipyard_db:
344 admin: shipyard-db-admin 358 admin: shipyard-db-admin
345 user: shipyard-db-user 359 user: shipyard-db-user
@@ -379,6 +393,17 @@ conf:
379 workflow_orchestrator:action_update_software: rule:admin_create 393 workflow_orchestrator:action_update_software: rule:admin_create
380 workflow_orchestrator:action_redeploy_server: rule:admin_create 394 workflow_orchestrator:action_redeploy_server: rule:admin_create
381 workflow_orchestrator:action_relabel_nodes: rule:admin_create 395 workflow_orchestrator:action_relabel_nodes: rule:admin_create
396 rabbitmq:
397 # adding rmq policy to mirror messages from celery queues
398 policies:
399 - vhost: "/"
400 name: "ha_celery"
401 definition:
402 ha-mode: "all"
403 ha-sync-mode: "automatic"
404 priority: 0
405 apply-to: all
406 pattern: 'celery.*'
382 paste: 407 paste:
383 app:shipyard-api: 408 app:shipyard-api:
384 paste.app_factory: shipyard_airflow.shipyard_api:paste_start_shipyard 409 paste.app_factory: shipyard_airflow.shipyard_api:paste_start_shipyard
@@ -760,6 +785,13 @@ pod:
760 memory: "128Mi" 785 memory: "128Mi"
761 cpu: "100m" 786 cpu: "100m"
762 jobs: 787 jobs:
788 rabbit_init:
789 limits:
790 memory: "128Mi"
791 cpu: "500m"
792 requests:
793 memory: "128Mi"
794 cpu: "500m"
763 airflow_db_init: 795 airflow_db_init:
764 limits: 796 limits:
765 memory: "128Mi" 797 memory: "128Mi"
@@ -839,6 +871,7 @@ manifests:
839 ingress_shipyard_api: true 871 ingress_shipyard_api: true
840 job_shipyard_db_init: true 872 job_shipyard_db_init: true
841 job_shipyard_db_sync: true 873 job_shipyard_db_sync: true
874 job_rabbit_init: true
842 job_airflow_db_init: true 875 job_airflow_db_init: true
843 job_airflow_db_sync: true 876 job_airflow_db_sync: true
844 job_ks_endpoints: true 877 job_ks_endpoints: true
@@ -848,6 +881,7 @@ manifests:
848 secret_shipyard_db: true 881 secret_shipyard_db: true
849 secret_ingress_tls: true 882 secret_ingress_tls: true
850 secret_keystone: true 883 secret_keystone: true
884 secret_rabbitmq: true
851 service_airflow_ingress: true 885 service_airflow_ingress: true
852 service_airflow_flower: true 886 service_airflow_flower: true
853 service_shipyard: true 887 service_shipyard: true
diff --git a/tools/gate/playbooks/zuul-linter.yaml b/tools/gate/playbooks/zuul-linter.yaml
index 7b6fb37..64a8926 100644
--- a/tools/gate/playbooks/zuul-linter.yaml
+++ b/tools/gate/playbooks/zuul-linter.yaml
@@ -15,6 +15,6 @@
15- hosts: primary 15- hosts: primary
16 tasks: 16 tasks:
17 - name: Execute a Whitespace Linter check 17 - name: Execute a Whitespace Linter check
18 command: find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -type f -exec egrep -l " +$" {} \; 18 command: find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -not -name "*.png" -type f -exec egrep -l " +$" {} \;
19 register: result 19 register: result
20 failed_when: result.stdout != "" \ No newline at end of file 20 failed_when: result.stdout != "" \ No newline at end of file