diff --git a/.gitignore b/.gitignore index 4289d053..d7f77b3e 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,6 @@ ENV/ # git Changelog AUTHORS + +# Ansible +*.retry diff --git a/.zuul.yaml b/.zuul.yaml index d93d0fb4..ee489bd6 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -19,17 +19,17 @@ - airship-deckhand-tox-py35-postgresql - airship-deckhand-ubuntu: voting: false - - airship-deckhand-functional-docker-py35 - airship-deckhand-functional-uwsgi-py35 + - airship-deckhand-functional-docker-py35 - airship-deckhand-integration-uwsgi-py35 + - airship-deckhand-integration-docker-py35 - airship-deckhand-docker-build-gate gate: jobs: - airship-deckhand-tox-py27-postgresql - airship-deckhand-tox-py35-postgresql - airship-deckhand-functional-docker-py35 - - airship-deckhand-functional-uwsgi-py35 - - airship-deckhand-integration-uwsgi-py35 + - airship-deckhand-integration-docker-py35 - airship-deckhand-docker-build-gate post: jobs: @@ -154,6 +154,42 @@ - ^releasenotes/.*$ - ^setup.cfg$ - ^deckhand/tests/unit/.*$ + vars: + disable_keystone: true + +- job: + name: airship-deckhand-integration-docker-base + description: | + Base job for running airship-deckhand integration tests. Runs tests + against Docker image generated from source code. + timeout: 3600 + roles: + - zuul: openstack/openstack-helm-infra + vars: + zuul_osh_relative_path: ../openstack-helm/ + zuul_osh_infra_relative_path: ../openstack-helm-infra/ + pre-run: + - tools/gate/playbooks/osh-infra-upgrade-host.yaml + - tools/gate/playbooks/osh-infra-deploy-docker.yaml + run: tools/gate/playbooks/run-integration-tests-docker.yaml + post-run: tools/gate/playbooks/osh-infra-collect-logs.yaml + required-projects: + - openstack/openstack-helm + - openstack/openstack-helm-infra + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ + - ^releasenotes/.*$ + +- job: + name: airship-deckhand-integration-docker-py35 + description: | + Run tox-based integration tests for the Airship Deckhand project under + cPython version 3.5. + parent: airship-deckhand-integration-docker-base + nodeset: openstack-helm-single-node + vars: + disable_keystone: false - job: name: airship-deckhand-docker-build-gate @@ -199,7 +235,7 @@ - job: name: airship-deckhand-doc-publish description: | - Publish documentation on airship-deckhand.readthedocs.io + Publish documentation on airship-deckhand.readthedocs.io. timeout: 300 run: tools/gate/playbooks/rtd-build-trigger.yaml nodeset: airship-deckhand-single-node diff --git a/tools/gate/playbooks/osh-infra-deploy-k8s.yaml b/tools/gate/playbooks/osh-infra-deploy-k8s.yaml index 5dc2ddeb..c6d630ad 100644 --- a/tools/gate/playbooks/osh-infra-deploy-k8s.yaml +++ b/tools/gate/playbooks/osh-infra-deploy-k8s.yaml @@ -33,4 +33,4 @@ roles: - deploy-kubeadm-aio-node tags: - - deploy-kubeadm-aio-node \ No newline at end of file + - deploy-kubeadm-aio-node diff --git a/tools/gate/playbooks/run-integration-tests-docker.yaml b/tools/gate/playbooks/run-integration-tests-docker.yaml new file mode 100644 index 00000000..65a8a0f5 --- /dev/null +++ b/tools/gate/playbooks/run-integration-tests-docker.yaml @@ -0,0 +1,34 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: all + roles: + - build-images + - deploy-keystone-dependencies + - deploy-barbican + # NOTE: deploy-postgresql and deploy-deckhand must follow + # deploy-keystone-dependencies which restarts Docker, causing + # the first 2 containers that are deployed to fail. + - deploy-postgresql + - generate-test-config + - deploy-deckhand + - run-integration-tests + tags: + - build-images + - deploy-keystone-dependencies + - deploy-barbican + - deploy-postgresql + - generate-test-config + - deploy-deckhand + - run-integration-tests diff --git a/tools/gate/roles/build-images/tasks/build-deckhand-image.yaml b/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml similarity index 93% rename from tools/gate/roles/build-images/tasks/build-deckhand-image.yaml rename to tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml index 46682bea..5742a971 100644 --- a/tools/gate/roles/build-images/tasks/build-deckhand-image.yaml +++ b/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml @@ -48,8 +48,6 @@ args: chdir: "{{ airship_deckhand_path.stdout }}" executable: /bin/bash - register: result - failed_when: "'permission denied' in result.stdout" become: yes - name: Build Airship-Deckhand Image Behind Proxy @@ -74,8 +72,6 @@ args: chdir: "{{ airship_deckhand_path.stdout }}" executable: /bin/bash - register: result - failed_when: "'permission denied' in result.stdout" become: yes - name: Register Deckhand Image ID @@ -83,7 +79,6 @@ set -e; echo $( docker images --filter label=zuul | awk '{print $3}' | head -2 | tail -1 ) register: airship_deckhand_image_id - failed_when: "'permission denied' in airship_deckhand_image_id.stdout" args: chdir: "{{ airship_deckhand_path.stdout }}" become: yes diff --git a/tools/gate/roles/build-images/tasks/main.yaml b/tools/gate/roles/build-images/tasks/main.yaml index 3f832344..a21029a5 100644 --- a/tools/gate/roles/build-images/tasks/main.yaml +++ b/tools/gate/roles/build-images/tasks/main.yaml @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -- include: build-deckhand-image.yaml +- include: build-airship-deckhand-image.yaml diff --git a/tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml b/tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml new file mode 100644 index 00000000..de330b67 --- /dev/null +++ b/tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml @@ -0,0 +1,46 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Deploy Barbican + shell: | + set -xe; + + # Pull images and lint chart + make pull-images barbican + + # Deploy command + helm upgrade --install barbican ./barbican \ + --namespace=openstack + + # Wait for deploy + ./tools/deployment/common/wait-for-pods.sh openstack + + # Validate deployment info + helm status barbican + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" + +- name: Retrieve Barbican URL for integration tests + shell: |- + set -e; + + test_barbican_url=$( + openstack endpoint list \ + --format value | grep barbican | grep public | awk '{print $7}' | sed 's/\(.*\)\//\1 /' + ) + + echo $test_barbican_url + register: test_barbican_url + environment: + OS_CLOUD: openstack_helm diff --git a/tools/gate/roles/deploy-barbican/tasks/main.yaml b/tools/gate/roles/deploy-barbican/tasks/main.yaml new file mode 100644 index 00000000..4e2e33c6 --- /dev/null +++ b/tools/gate/roles/deploy-barbican/tasks/main.yaml @@ -0,0 +1,15 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: deploy-barbican.yaml diff --git a/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml b/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml index 14f99a8a..f452a749 100644 --- a/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml +++ b/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml @@ -12,32 +12,67 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Deploy Deckhand using Docker +- name: Perform Database Migrations shell: |- set -ex; - echo "Running Deckhand via Docker." - # Do database migrations. docker run \ - --rm \ - --net=host \ - -v "{{ deckhand_conf_dir.stdout }}":/etc/deckhand \ - "{{ airship_deckhand_image_id.stdout }}" alembic upgrade head & + --rm \ + --net=host \ + -v "{{ deckhand_conf_dir }}":/etc/deckhand "{{ airship_deckhand_image_id.stdout }}" alembic upgrade head # Allow migrations to complete. - sleep 5 + sleep 10 + become: yes - # Start Deckhand container. +- name: Deploy Deckhand Container + shell: |- docker run \ - --rm \ - --net=host \ - -p 9000:9000 \ - -v "{{ deckhand_conf_dir.stdout }}":/etc/deckhand \ - "{{ airship_deckhand_image_id.stdout }}" server & + --rm \ + --net=host \ + -p 9000:9000 \ + -v "{{ deckhand_conf_dir }}":/etc/deckhand "{{ airship_deckhand_image_id.stdout }}" server & # Give the server a chance to come up. Better to poll a health check. - sleep 5 + sleep 10 args: chdir: "{{ airship_deckhand_path.stdout }}" become: yes + +- name: Register Deckhand Service and Endpoints + when: disable_keystone == false + block: + - name: Register Keystone Deckhand Service + shell: |- + set -ex; + + if [ -z "$( openstack service list --format value | grep deckhand )" ]; then + openstack service create --enable --name deckhand deckhand + fi + environment: + OS_CLOUD: openstack_helm + register: result + failed_when: "'permission denied' in result.stdout" + + - name: Register Keystone Deckhand Endpoints + shell: |- + set -ex; + + echo "Registering Deckhand endpoint: {{ item }}" + + deckhand_endpoint="http://127.0.0.1:9000" + + if [ -z "$( openstack endpoint list --format value | grep deckhand | grep {{ item }} )" ]; then + openstack endpoint create --enable \ + --region RegionOne \ + deckhand "{{ item }}" $deckhand_endpoint/api/v1.0 + fi + with_items: + - "admin" + - "public" + - "internal" + environment: + OS_CLOUD: openstack_helm + register: result + failed_when: "'permission denied' in result.stdout" diff --git a/tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml b/tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml new file mode 100644 index 00000000..e141cb9f --- /dev/null +++ b/tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml @@ -0,0 +1,85 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set zuul_site_mirror_fqdn from env var if not defined + when: zuul_site_mirror_fqdn is not defined + set_fact: + zuul_site_mirror_fqdn: "{{ lookup('env','zuul_site_mirror_fqdn') }}" + +- name: Deploy Required Packages + shell: | + set -xe; + ./tools/deployment/developer/nfs/000-install-packages.sh + environment: + zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" + +- name: Deploy Kubernetes + shell: | + set -xe; + ./tools/deployment/developer/nfs/010-deploy-k8s.sh + environment: + zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" + +- name: Setup OS and Kubernetes Clients + shell: | + set -xe; + ./tools/deployment/developer/nfs/020-setup-client.sh + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" + +- name: Deploy Ingress + shell: | + set -xe; + ./tools/deployment/developer/nfs/030-ingress.sh + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" + +- name: Deploy NFS + shell: | + set -xe; + ./tools/deployment/developer/nfs/040-nfs-provisioner.sh + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" + +- name: Deploy MariaDB + shell: | + set -xe; + ./tools/deployment/developer/nfs/050-mariadb.sh + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" + +- name: Deploy RabbitMQ + shell: | + set -xe; + ./tools/deployment/developer/nfs/060-rabbitmq.sh + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" + +- name: Deploy Memcached + shell: | + set -xe; + ./tools/deployment/developer/nfs/070-memcached.sh + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" + +- name: Deploy Keystone + shell: | + set -xe; + ./tools/deployment/developer/nfs/080-keystone.sh + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" diff --git a/tools/gate/roles/deploy-keystone-dependencies/tasks/main.yaml b/tools/gate/roles/deploy-keystone-dependencies/tasks/main.yaml new file mode 100644 index 00000000..de379b4a --- /dev/null +++ b/tools/gate/roles/deploy-keystone-dependencies/tasks/main.yaml @@ -0,0 +1,15 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: deploy-keystone-dependencies.yaml diff --git a/tools/gate/roles/generate-test-config/tasks/generate-test-config.yaml b/tools/gate/roles/generate-test-config/tasks/generate-test-config.yaml index 81858801..257abb92 100644 --- a/tools/gate/roles/generate-test-config/tasks/generate-test-config.yaml +++ b/tools/gate/roles/generate-test-config/tasks/generate-test-config.yaml @@ -13,27 +13,27 @@ # limitations under the License. - name: Create temporary deckhand conf directory - shell: |- - set - ex; - echo $( mktemp -d -p "{{ airship_deckhand_path.stdout }}" ) - register: deckhand_conf_dir + tempfile: + state: directory + suffix: airship-deckhand + register: _deckhand_conf_dir + +- name: Store deckhand_conf_dir in variable + set_fact: + deckhand_conf_dir: "{{ _deckhand_conf_dir.path }}" environment: # Used by Deckhand's initialization script to search for config files. - AIRSHIP_DECKHAND_CONFIG_DIR: "{{ deckhand_conf_dir.stdout }}" - args: - chdir: "{{ airship_deckhand_path.stdout }}" + AIRSHIP_DECKHAND_CONFIG_DIR: "{{ deckhand_conf_dir }}" - name: Generate test deckhand.conf file when disable_keystone is true when: disable_keystone == true shell: |- set -ex; - chmod 777 -R "{{ deckhand_conf_dir.stdout }}" - conf_file="{{ deckhand_conf_dir.stdout }}"/deckhand.conf + chmod 777 -R "{{ deckhand_conf_dir }}" + conf_file="{{ deckhand_conf_dir }}"/deckhand.conf - echo "{{ airship_deckhand_database_url }}" - - cp etc/deckhand/logging.conf.sample "{{ deckhand_conf_dir.stdout }}"/logging.conf + cp etc/deckhand/logging.conf.sample "{{ deckhand_conf_dir }}"/logging.conf envsubst '$AIRSHIP_DECKHAND_DATABASE_URL' < deckhand/tests/deckhand.conf.test > $conf_file echo "Toggling development_mode on to disable Keystone authentication." @@ -42,8 +42,8 @@ echo $conf_file 1>&2 cat $conf_file 1>&2 - echo "{{ deckhand_conf_dir.stdout }}"/logging.conf 1>&2 - cat "{{ deckhand_conf_dir.stdout }}"/logging.conf 1>&2 + echo "{{ deckhand_conf_dir }}"/logging.conf 1>&2 + cat "{{ deckhand_conf_dir }}"/logging.conf 1>&2 args: chdir: "{{ airship_deckhand_path.stdout }}" environment: @@ -55,18 +55,20 @@ shell: |- set -ex; - conf_file="{{ deckhand_conf_dir.stdout }}"/deckhand.conf + chmod 777 -R "{{ deckhand_conf_dir }}" + conf_file="{{ deckhand_conf_dir }}"/deckhand.conf - cp etc/deckhand/logging.conf.sample "{{ deckhand_conf_dir.stdout }}"/logging.conf - envsubst '$AIRSHIP_DECKHAND_DATABASE_URL' < deckhand/tests/deckhand.conf.test > $conf_file + cp etc/deckhand/logging.conf.sample "{{ deckhand_conf_dir }}"/logging.conf + envsubst '$AIRSHIP_DECKHAND_DATABASE_URL $TEST_BARBICAN_URL' < deckhand/tests/deckhand.conf.test > $conf_file echo $conf_file 1>&2 cat $conf_file 1>&2 - echo "{{ deckhand_conf_dir.stdout }}"/logging.conf 1>&2 - cat "{{ deckhand_conf_dir.stdout }}"/logging.conf 1>&2 + echo "{{ deckhand_conf_dir }}"/logging.conf 1>&2 + cat "{{ deckhand_conf_dir }}"/logging.conf 1>&2 args: chdir: "{{ airship_deckhand_path.stdout }}" environment: AIRSHIP_DECKHAND_DATABASE_URL: "{{ airship_deckhand_database_url }}" + TEST_BARBICAN_URL: "{{ test_barbican_url.stdout }}" become: yes diff --git a/tools/gate/roles/generate-test-config/tasks/generate-test-paste.yaml b/tools/gate/roles/generate-test-config/tasks/generate-test-paste.yaml index 65be6f04..c9206411 100644 --- a/tools/gate/roles/generate-test-config/tasks/generate-test-paste.yaml +++ b/tools/gate/roles/generate-test-config/tasks/generate-test-paste.yaml @@ -17,7 +17,7 @@ shell: |- set -ex; echo "Using noauth-paste.ini to disable Keystone authentication." - cp etc/deckhand/noauth-paste.ini "{{ deckhand_conf_dir.stdout }}"/noauth-paste.ini + cp etc/deckhand/noauth-paste.ini "{{ deckhand_conf_dir }}"/noauth-paste.ini args: chdir: "{{ airship_deckhand_path.stdout }}" @@ -25,6 +25,6 @@ when: disable_keystone == false shell: |- set -ex; - cp etc/deckhand/deckhand-paste.ini "{{ deckhand_conf_dir.stdout }}"/deckhand-paste.ini + cp etc/deckhand/deckhand-paste.ini "{{ deckhand_conf_dir }}"/deckhand-paste.ini args: chdir: "{{ airship_deckhand_path.stdout }}" diff --git a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml index 03118b53..d40b0bab 100644 --- a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml +++ b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml @@ -12,7 +12,48 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Run Integration Tests +- name: Install pip3 and gabbi + shell: | + set -xe; + apt-get install -y python3-pip + pip3 install --upgrade pip + # Includes required requirements for testing. + pip3 install -r test-requirements.txt + args: + chdir: "{{ zuul.project.src_dir }}" + become: yes + +- name: Run Integration Tests with Docker + when: disable_keystone == false + block: + - name: Generate Keystone token for integration tests + shell: |- + set -e; + echo $( openstack token issue --format value -c id ) + args: + chdir: "{{ zuul.project.src_dir }}" + register: test_auth_token + environment: + OS_CLOUD: openstack_helm + + - name: Run Integration Tests + shell: | + set -ex; + py.test -svx "{{ airship_deckhand_path.stdout }}"/deckhand/tests/common/test_gabbi.py \ + && echo "Done SUCCESS" || echo "Done FAILURE" + args: + chdir: "{{ zuul.project.src_dir }}" + environment: + DECKHAND_IMAGE: "{{ airship_deckhand_image_id.stdout }}" + DECKHAND_TEST_URL: "127.0.0.1:9000" + DECKHAND_TEST_DIR: "{{ airship_deckhand_path.stdout }}/deckhand/tests/integration/gabbits" + TEST_AUTH_TOKEN: "{{ test_auth_token.stdout }}" + TEST_BARBICAN_URL: "{{ test_barbican_url.stdout }}" + register: result + failed_when: "result.rc != 0" + +- name: Run Integration Tests with uwsgi + when: disable_keystone == true shell: | set -xe; # NOTE(felipemonteiro): We don't use a venv because they don't play nicely @@ -27,4 +68,4 @@ DECKHAND_TEST_URL: "127.0.0.1:9000" DECKHAND_TEST_DIR: "{{ airship_deckhand_path.stdout }}/deckhand/tests/integration/gabbits" register: result - failed_when: "'commands failed' in result.stdout" + failed_when: "result.rc != 0"