diff --git a/charts/shipyard/templates/configmap-airflow-bin.yaml b/charts/shipyard/templates/configmap-airflow-bin.yaml index c19562be..12ab0f1f 100644 --- a/charts/shipyard/templates/configmap-airflow-bin.yaml +++ b/charts/shipyard/templates/configmap-airflow-bin.yaml @@ -21,6 +21,8 @@ kind: ConfigMap metadata: name: airflow-bin data: + rabbit-init.sh: |+ +{{- include "helm-toolkit.scripts.rabbit_init" . | indent 4 }} airflow-db-init.sh: |+ {{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} airflow-db-sync.sh: |+ diff --git a/charts/shipyard/templates/job-rabbit-init.yaml b/charts/shipyard/templates/job-rabbit-init.yaml new file mode 100644 index 00000000..39aa527d --- /dev/null +++ b/charts/shipyard/templates/job-rabbit-init.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_rabbit_init }} +{{- $rmqJob := dict "envAll" . "serviceName" "airflow" -}} +{{ $rmqJob | include "helm-toolkit.manifests.job_rabbit_init" }} +{{- end }} \ No newline at end of file diff --git a/charts/shipyard/templates/secret-rabbitmq.yaml b/charts/shipyard/templates/secret-rabbitmq.yaml new file mode 100644 index 00000000..235ae22f --- /dev/null +++ b/charts/shipyard/templates/secret-rabbitmq.yaml @@ -0,0 +1,28 @@ +{{/* +Copyright 2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_rabbitmq }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.oslo_messaging "admin" }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + RABBITMQ_CONNECTION: {{ tuple "oslo_messaging" "internal" "admin" "http" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc }} +{{- end }} diff --git a/charts/shipyard/templates/secret-user-rabbitmq.yaml b/charts/shipyard/templates/secret-user-rabbitmq.yaml new file mode 100644 index 00000000..c69795d7 --- /dev/null +++ b/charts/shipyard/templates/secret-user-rabbitmq.yaml @@ -0,0 +1,28 @@ +{{/* +Copyright 2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_rabbitmq }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.oslo_messaging "airflow" }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + RABBITMQ_CONNECTION: {{ tuple "oslo_messaging" "internal" "user" "amqp" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc }} +{{- end }} \ No newline at end of file diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 6688bd8f..3c88bd88 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -37,6 +37,7 @@ images: shipyard_db_init: docker.io/postgres:9.5 shipyard_db_sync: quay.io/airshipit/shipyard:latest airflow_db_init: docker.io/postgres:9.5 + rabbit_init: docker.io/rabbitmq:3.7-management airflow_db_sync: quay.io/airshipit/airflow:latest ks_user: docker.io/openstackhelm/heat:ocata ks_service: docker.io/openstackhelm/heat:ocata @@ -105,6 +106,10 @@ dependencies: services: - service: postgresql_airflow_db endpoint: internal + rabbit_init: + services: + - service: oslo_messaging + endpoint: internal airflow_db_sync: jobs: - airflow-db-init @@ -143,6 +148,7 @@ dependencies: endpoint: internal airflow_server: jobs: + - airflow-rabbit-init - airflow-db-init - airflow-db-sync services: @@ -316,17 +322,22 @@ endpoints: oslo_messaging: auth: user: + username: airflow + password: password + admin: username: rabbitmq password: password hosts: default: rabbitmq host_fqdn_override: default: null - path: / + path: /airflow scheme: amqp port: amqp: default: 5672 + http: + default: 15672 oslo_cache: hosts: default: memcached @@ -340,6 +351,9 @@ secrets: identity: admin: shipyard-keystone-admin shipyard: shipyard-keystone-user + oslo_messaging: + admin: airflow-rabbitmq-admin + airflow: airflow-rabbitmq-user postgresql_shipyard_db: admin: shipyard-db-admin user: shipyard-db-user @@ -381,6 +395,17 @@ conf: workflow_orchestrator:action_update_software: rule:admin_create workflow_orchestrator:action_redeploy_server: rule:admin_create workflow_orchestrator:action_relabel_nodes: rule:admin_create + rabbitmq: + # adding rmq policy to mirror messages from celery queues + policies: + - vhost: "/" + name: "ha_celery" + definition: + ha-mode: "all" + ha-sync-mode: "automatic" + priority: 0 + apply-to: all + pattern: 'celery.*' paste: app:shipyard-api: paste.app_factory: shipyard_airflow.shipyard_api:paste_start_shipyard @@ -785,6 +810,13 @@ pod: memory: "128Mi" cpu: "100m" jobs: + rabbit_init: + limits: + memory: "128Mi" + cpu: "500m" + requests: + memory: "128Mi" + cpu: "500m" airflow_db_init: limits: memory: "128Mi" @@ -864,6 +896,7 @@ manifests: ingress_shipyard_api: true job_shipyard_db_init: true job_shipyard_db_sync: true + job_rabbit_init: true job_airflow_db_init: true job_airflow_db_sync: true job_ks_endpoints: true @@ -873,6 +906,7 @@ manifests: secret_shipyard_db: true secret_ingress_tls: true secret_keystone: true + secret_rabbitmq: true service_airflow_ingress: true service_airflow_flower: true service_shipyard: true diff --git a/tools/gate/playbooks/zuul-linter.yaml b/tools/gate/playbooks/zuul-linter.yaml index 7b6fb37f..64a89268 100644 --- a/tools/gate/playbooks/zuul-linter.yaml +++ b/tools/gate/playbooks/zuul-linter.yaml @@ -15,6 +15,6 @@ - hosts: primary tasks: - name: Execute a Whitespace Linter check - command: find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -type f -exec egrep -l " +$" {} \; + command: find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -not -name "*.png" -type f -exec egrep -l " +$" {} \; register: result failed_when: result.stdout != "" \ No newline at end of file