From eadae6d7b167c73d68d56d13e810205530018012 Mon Sep 17 00:00:00 2001 From: "Crank, Daniel (dc6350)" Date: Thu, 25 Apr 2019 15:14:23 -0500 Subject: [PATCH] Fix Deckhand integration test gates Fix problems with integration test zuul jobs which were made non-voting by [0] and restore them to voting status. These jobs exercise some secret-management functions that test integration between Deckhand and Barbican that doesn't happen in the Airskiff jobs, so they still have some test value. - Some integration test scripts were pointing to old versions of openstack-helm scripts that are no longer maintained. This PS updates those scripts to point to current OSH scripts. - Uplift openstack-helm-infra pin in helm_tk.sh to a recent version - Add build-images to uwsgi test jobs to ensure airship_deckhand_path is set [0]: https://review.opendev.org/660738 Change-Id: If83dead126307a98b44ad5d8f367a972c48a30d0 --- .zuul.yaml | 8 +++--- test-requirements.txt | 1 + .../run-integration-tests-docker.yaml | 1 + .../tasks/deploy-keystone-dependencies.yaml | 18 ++++++------ .../tasks/install-test-requirements.yaml | 28 +++++++++++++++++++ .../install-test-requirements/tasks/main.yaml | 15 ++++++++++ .../tasks/integration-tests.yaml | 10 ------- tools/helm_tk.sh | 2 +- tools/integration-tests.sh | 26 +++++++++-------- 9 files changed, 73 insertions(+), 36 deletions(-) create mode 100644 tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml create mode 100644 tools/gate/roles/install-test-requirements/tasks/main.yaml diff --git a/.zuul.yaml b/.zuul.yaml index 07482ef2..60155a6b 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -29,10 +29,8 @@ - deckhand-functional-docker-py35-opensuse - deckhand-integration-uwsgi-py35: voting: false - - deckhand-integration-docker-py35-ubuntu: - voting: false - - deckhand-integration-docker-py35-opensuse: - voting: false + - deckhand-integration-docker-py35-ubuntu + - deckhand-integration-docker-py35-opensuse - deckhand-chart-build-gate - deckhand-chart-build-latest-htk - deckhand-docker-build-gate-ubuntu @@ -43,6 +41,8 @@ - deckhand-tox-py35-postgresql - deckhand-functional-docker-py35-ubuntu - deckhand-functional-docker-py35-opensuse + - deckhand-integration-docker-py35-ubuntu + - deckhand-integration-docker-py35-opensuse - deckhand-chart-build-gate - deckhand-docker-build-gate-ubuntu - deckhand-docker-build-gate-opensuse diff --git a/test-requirements.txt b/test-requirements.txt index daa18e64..93bfaa82 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,3 +15,4 @@ bandit==1.5.1 gabbi==1.35.1 pifpaf==2.1.2 oslotest==3.7.0 +yq>=2.7.2 \ No newline at end of file diff --git a/tools/gate/playbooks/run-integration-tests-docker.yaml b/tools/gate/playbooks/run-integration-tests-docker.yaml index 1044ad0d..6cf84121 100644 --- a/tools/gate/playbooks/run-integration-tests-docker.yaml +++ b/tools/gate/playbooks/run-integration-tests-docker.yaml @@ -14,6 +14,7 @@ - hosts: all roles: + - install-test-requirements - build-images - deploy-keystone-dependencies - build-charts diff --git a/tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml b/tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml index e141cb9f..00c57dd1 100644 --- a/tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml +++ b/tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml @@ -20,7 +20,7 @@ - name: Deploy Required Packages shell: | set -xe; - ./tools/deployment/developer/nfs/000-install-packages.sh + ./tools/deployment/common/install-packages.sh environment: zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" args: @@ -29,7 +29,7 @@ - name: Deploy Kubernetes shell: | set -xe; - ./tools/deployment/developer/nfs/010-deploy-k8s.sh + ./tools/deployment/common/deploy-k8s.sh environment: zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" args: @@ -38,48 +38,48 @@ - name: Setup OS and Kubernetes Clients shell: | set -xe; - ./tools/deployment/developer/nfs/020-setup-client.sh + ./tools/deployment/common/setup-client.sh args: chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" - name: Deploy Ingress shell: | set -xe; - ./tools/deployment/developer/nfs/030-ingress.sh + ./tools/deployment/component/common/ingress.sh args: chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" - name: Deploy NFS shell: | set -xe; - ./tools/deployment/developer/nfs/040-nfs-provisioner.sh + ./tools/deployment/component/nfs-provisioner/nfs-provisioner.sh args: chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" - name: Deploy MariaDB shell: | set -xe; - ./tools/deployment/developer/nfs/050-mariadb.sh + ./tools/deployment/component/common/mariadb.sh args: chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" - name: Deploy RabbitMQ shell: | set -xe; - ./tools/deployment/developer/nfs/060-rabbitmq.sh + ./tools/deployment/component/common/rabbitmq.sh args: chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" - name: Deploy Memcached shell: | set -xe; - ./tools/deployment/developer/nfs/070-memcached.sh + ./tools/deployment/component/common/memcached.sh args: chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" - name: Deploy Keystone shell: | set -xe; - ./tools/deployment/developer/nfs/080-keystone.sh + ./tools/deployment/component/keystone/keystone.sh args: chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" diff --git a/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml b/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml new file mode 100644 index 00000000..44583c03 --- /dev/null +++ b/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml @@ -0,0 +1,28 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set modprobe br_netfilter + shell: | + set -xe; + sudo modprobe br_netfilter + +- name: Install pip3 and gabbi + shell: | + set -xe; + apt-get install -y python-pip + pip install --upgrade pip + pip install -r test-requirements.txt + args: + chdir: "{{ zuul.project.src_dir }}" + become: yes diff --git a/tools/gate/roles/install-test-requirements/tasks/main.yaml b/tools/gate/roles/install-test-requirements/tasks/main.yaml new file mode 100644 index 00000000..ee0a62c2 --- /dev/null +++ b/tools/gate/roles/install-test-requirements/tasks/main.yaml @@ -0,0 +1,15 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: install-test-requirements.yaml diff --git a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml index 3a62e615..48d12a99 100644 --- a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml +++ b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml @@ -12,16 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Install pip3 and gabbi - shell: | - set -xe; - apt-get install -y python-pip - pip install --upgrade pip - pip install -r test-requirements.txt - args: - chdir: "{{ zuul.project.src_dir }}" - become: yes - - name: Run Integration Tests with Docker when: disable_keystone == false block: diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh index ffe5538e..c21afcfc 100755 --- a/tools/helm_tk.sh +++ b/tools/helm_tk.sh @@ -18,7 +18,7 @@ HELM=$1 HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"} HTK_PATH=${HTK_PATH:-""} -HTK_STABLE_COMMIT=${HTK_COMMIT:-"200b5e902b3a176fbfbe669b6a10a254c9b50f5d"} +HTK_STABLE_COMMIT=${HTK_COMMIT:-"926348fe2476051f9ca825916db333f81c0139b7"} BUILD_DIR=${BUILD_DIR:-$(mktemp -d)} if [[ ! -z $(echo $http_proxy) ]] diff --git a/tools/integration-tests.sh b/tools/integration-tests.sh index 251b3d67..b5368692 100755 --- a/tools/integration-tests.sh +++ b/tools/integration-tests.sh @@ -46,29 +46,31 @@ function deploy_osh_keystone_barbican { git clone https://git.openstack.org/openstack/openstack-helm.git ../openstack-helm fi - cd ${OSH_INFRA_PATH} - # NOTE(fmontei): setup-host already sets up required host dependencies. - make dev-deploy setup-host - make dev-deploy k8s + cd ${OSH_PATH} + # Deploy required packages + ./tools/deployment/common/install-packages.sh + # Deploy Kubernetes + sudo modprobe br_netfilter + ./tools/deployment/common/deploy-k8s.sh cd ${CURRENT_DIR} sudo -H -E pip install -r test-requirements.txt cd ${OSH_PATH} - # Setup clients on the host and assemble the charts¶ - ./tools/deployment/developer/common/020-setup-client.sh + # Setup clients on the host and assemble the charts + ./tools/deployment/common/setup-client.sh # Deploy the ingress controller - ./tools/deployment/developer/common/030-ingress.sh + ./tools/deployment/component/common/ingress.sh # Deploy NFS Provisioner - ./tools/deployment/developer/nfs/040-nfs-provisioner.sh + ./tools/deployment/component/nfs-provisioner/nfs-provisioner.sh # Deploy MariaDB - ./tools/deployment/developer/nfs/050-mariadb.sh + ./tools/deployment/component/common/mariadb.sh # Deploy RabbitMQ - ./tools/deployment/developer/nfs/060-rabbitmq.sh + ./tools/deployment/component/common/rabbitmq.sh # Deploy Memcached - ./tools/deployment/developer/nfs/070-memcached.sh + ./tools/deployment/component/common/memcached.sh # Deploy Keystone - ./tools/deployment/developer/nfs/080-keystone.sh + ./tools/deployment/component/keystone/keystone.sh deploy_barbican }