From e7ba6828a0a1ca27fae596f6e0ee5a857f28001d Mon Sep 17 00:00:00 2001 From: Ahmad Mahmoudi Date: Thu, 20 Feb 2020 19:25:02 +0000 Subject: [PATCH] (fix) Address uwsgi and other gating issues - With Ubuntu bionic based deckhand docker image, uswsi crashes with segmentation fault when it tries to load psycopg2 library, causing the deckhand service become unavailable. The root cause of this problem seems to be that uwsgi and psycopg2 binary wheels are built with different ssl libraries. To address this issue, upgrading psycopg2 to the latest release to the latest binary release. - For opensuse 15.1 image build, python3-six was updated to address package not found issue. - Updated gating ansbile playbooks to address non-voting gate failures. Change-Id: I7be920e16e6114eb2bdbc052a6761f29008baf81 --- .zuul.yaml | 42 ++++--------------- images/deckhand/Dockerfile.opensuse_15 | 2 +- requirements.txt | 4 +- .../playbooks/run-functional-tests-uwsgi.yaml | 1 + .../run-integration-tests-uwsgi.yaml | 1 + .../tasks/disable-systemd-resolved.yaml | 37 ++++++++++++++++ .../disable-systemd-resolved/tasks/main.yaml | 15 +++++++ .../tasks/functional-tests.yaml | 6 +++ .../tasks/integration-tests.yaml | 6 +++ tools/integration-tests.sh | 8 ++++ 10 files changed, 86 insertions(+), 36 deletions(-) create mode 100644 tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml create mode 100644 tools/gate/roles/disable-systemd-resolved/tasks/main.yaml diff --git a/.zuul.yaml b/.zuul.yaml index f1bc554b..c754dc27 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -93,7 +93,7 @@ nodeset: deckhand-single-node vars: tox_envlist: functional-dev - irrelevant-files: + irrelevant-files: &irrelevant-files - ^.*\.rst$ - ^doc/.*$ - ^etc/.*$ @@ -121,40 +121,28 @@ description: | Run tox-based functional tests for the Airship Deckhand project under cPython version 3.5. Uses tox with the ``functional-py35`` environment. - Ubuntu (default) image is built and used. + Ubuntu (xenial) image is built and used. parent: deckhand-functional-docker-base nodeset: deckhand-single-node vars: tox_envlist: functional disable_keystone: true distro: ubuntu_xenial - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - ^etc/.*$ - - ^releasenotes/.*$ - - ^setup.cfg$ - - ^deckhand/tests/unit/.*$ + irrelevant-files: *irrelevant-files - job: name: deckhand-functional-docker-py35-ubuntu_bionic description: | Run tox-based functional tests for the Airship Deckhand project under cPython version 3.5. Uses tox with the ``functional-py35`` environment. - Ubuntu (default) image is built and used. + Ubuntu (bionic) image is built and used. parent: deckhand-functional-docker-base nodeset: deckhand-single-node vars: tox_envlist: functional disable_keystone: true distro: ubuntu_bionic - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - ^etc/.*$ - - ^releasenotes/.*$ - - ^setup.cfg$ - - ^deckhand/tests/unit/.*$ + irrelevant-files: *irrelevant-files - job: name: deckhand-functional-docker-py35-opensuse @@ -168,13 +156,7 @@ tox_envlist: functional disable_keystone: true distro: opensuse_15 - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - ^etc/.*$ - - ^releasenotes/.*$ - - ^setup.cfg$ - - ^deckhand/tests/unit/.*$ + irrelevant-files: *irrelevant-files - job: name: deckhand-integration-uwsgi-py35 @@ -185,13 +167,7 @@ timeout: 3600 run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml nodeset: deckhand-single-node - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - ^etc/.*$ - - ^releasenotes/.*$ - - ^setup.cfg$ - - ^deckhand/tests/unit/.*$ + irrelevant-files: *irrelevant-files vars: disable_keystone: true @@ -242,7 +218,7 @@ name: deckhand-integration-docker-py35-ubuntu_xenial description: | Run tox-based integration tests for the Airship Deckhand project under - cPython version 3.5. Builds ubuntu (default) deckhand image. + cPython version 3.5. Builds ubuntu (xenial) deckhand image. parent: deckhand-integration-docker-base nodeset: openstack-helm-single-node vars: @@ -253,7 +229,7 @@ name: deckhand-integration-docker-py35-ubuntu_bionic description: | Run tox-based integration tests for the Airship Deckhand project under - cPython version 3.5. Builds ubuntu (default) deckhand image. + cPython version 3.5. Builds ubuntu (bionic) deckhand image. parent: deckhand-integration-docker-base nodeset: openstack-helm-single-node vars: diff --git a/images/deckhand/Dockerfile.opensuse_15 b/images/deckhand/Dockerfile.opensuse_15 index be9b25bc..87aaa1d9 100644 --- a/images/deckhand/Dockerfile.opensuse_15 +++ b/images/deckhand/Dockerfile.opensuse_15 @@ -50,7 +50,7 @@ RUN set -x && \ libpqxx-devel \ && zypper --non-interactive ar -f https://download.opensuse.org/repositories/devel:languages:python/openSUSE_Leap_15.1/devel:languages:python.repo \ && zypper --non-interactive --gpg-auto-import-keys refresh devel_languages_python \ - && zypper --non-interactive install python3-six-1.12.0-lp151.80.1.noarch + && zypper --non-interactive install python3-six-1.11.0-lp151.3.1.noarch RUN pip3 install -U pip \ && zypper clean -a \ diff --git a/requirements.txt b/requirements.txt index 084bcaca..ea22a666 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ oslo.utils==3.40.2 pbr==5.1.0 PasteDeploy==1.5.2 Paste==3.0.1 -psycopg2==2.7.5 +psycopg2-binary==2.8.4 pyyaml~=5.1 # TODO(alanmeadows) @@ -40,7 +40,7 @@ python-keystoneclient==3.18.0 python-memcached==1.59 Routes==2.4.1 -six==1.12.0 +six==1.11.0 stevedore==1.30.0 urllib3==1.24.3 uwsgi==2.0.17.1 diff --git a/tools/gate/playbooks/run-functional-tests-uwsgi.yaml b/tools/gate/playbooks/run-functional-tests-uwsgi.yaml index 7ba32e62..e28d03e3 100644 --- a/tools/gate/playbooks/run-functional-tests-uwsgi.yaml +++ b/tools/gate/playbooks/run-functional-tests-uwsgi.yaml @@ -15,6 +15,7 @@ - hosts: all gather_facts: False roles: + - disable-systemd-resolved - install-postgresql - run-functional-tests tags: diff --git a/tools/gate/playbooks/run-integration-tests-uwsgi.yaml b/tools/gate/playbooks/run-integration-tests-uwsgi.yaml index 9160c412..10bc32be 100644 --- a/tools/gate/playbooks/run-integration-tests-uwsgi.yaml +++ b/tools/gate/playbooks/run-integration-tests-uwsgi.yaml @@ -15,6 +15,7 @@ - hosts: all gather_facts: False roles: + - disable-systemd-resolved - install-postgresql - run-integration-tests tags: diff --git a/tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml b/tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml new file mode 100644 index 00000000..0eda1fbf --- /dev/null +++ b/tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml @@ -0,0 +1,37 @@ +# Copyright 2020 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Disable systemd-resolved service + systemd: + state: stopped + enabled: no + masked: yes + daemon_reload: yes + name: systemd-resolved + become: yes + +- name: Remove local stub dns from resolv.conf, if it exists + lineinfile: + path: /etc/resolv.conf + state: absent + regexp: '^nameserver.*127.0.0.1' + become: yes + +- name: Add upstream nameservers in resolv.conf + blockinfile: + path: /etc/resolv.conf + block: | + nameserver 8.8.8.8 + nameserver 8.8.4.4 + become: yes diff --git a/tools/gate/roles/disable-systemd-resolved/tasks/main.yaml b/tools/gate/roles/disable-systemd-resolved/tasks/main.yaml new file mode 100644 index 00000000..bb381b41 --- /dev/null +++ b/tools/gate/roles/disable-systemd-resolved/tasks/main.yaml @@ -0,0 +1,15 @@ +# Copyright 2020 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: disable-systemd-resolved.yaml diff --git a/tools/gate/roles/run-functional-tests/tasks/functional-tests.yaml b/tools/gate/roles/run-functional-tests/tasks/functional-tests.yaml index 500a2e87..1b9d1bff 100644 --- a/tools/gate/roles/run-functional-tests/tasks/functional-tests.yaml +++ b/tools/gate/roles/run-functional-tests/tasks/functional-tests.yaml @@ -12,6 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Airship-Deckhand image build path + shell: cd "{{ work_dir }}"; pwd + register: airship_deckhand_path + vars: + work_dir: "{{ zuul.project.src_dir | default('') }}/{{ zuul_airship_deckhand_relative_path | default('') }}" + - name: Run Functional Tests shell: | set -xe; diff --git a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml index 48d12a99..b63dad5c 100644 --- a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml +++ b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml @@ -12,6 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Airship-Deckhand image build path + shell: cd "{{ work_dir }}"; pwd + register: airship_deckhand_path + vars: + work_dir: "{{ zuul.project.src_dir | default('') }}/{{ zuul_airship_deckhand_relative_path | default('') }}" + - name: Run Integration Tests with Docker when: disable_keystone == false block: diff --git a/tools/integration-tests.sh b/tools/integration-tests.sh index b5368692..72327aa2 100755 --- a/tools/integration-tests.sh +++ b/tools/integration-tests.sh @@ -56,6 +56,14 @@ function deploy_osh_keystone_barbican { cd ${CURRENT_DIR} sudo -H -E pip install -r test-requirements.txt + # remove systemd-resolved local stub dns from resolv.conf + sudo sed -i.bkp '/^nameserver.*127.0.0.1/d + w /dev/stdout' /etc/resolv.conf + # add external nameservers + echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf + echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf + cat /etc/resolv.conf + cd ${OSH_PATH} # Setup clients on the host and assemble the charts ./tools/deployment/common/setup-client.sh