diff --git a/.zuul.yaml b/.zuul.yaml index 9675be18..89b8fd17 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -126,6 +126,7 @@ - airship/treasuremap vars: CLONE_SHIPYARD: false + HTK_COMMIT: master irrelevant-files: &irrelevant-files - ^.*\.rst$ - ^doc/.*$ diff --git a/images/airflow/Dockerfile.ubuntu_xenial b/images/airflow/Dockerfile.ubuntu_xenial index da20d03a..aaad425d 100644 --- a/images/airflow/Dockerfile.ubuntu_xenial +++ b/images/airflow/Dockerfile.ubuntu_xenial @@ -69,13 +69,13 @@ RUN set -ex && \ netcat \ netbase \ python3 \ - python3-setuptools \ python3-pip \ python3-dev \ python3-dateutil \ make \ --no-install-recommends \ - && python3 -m pip install -U pip \ + && python3 -m pip install -U 'pip<21.0' \ + && python3 -m pip install -U setuptools \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ diff --git a/images/shipyard/Dockerfile.ubuntu_xenial b/images/shipyard/Dockerfile.ubuntu_xenial index 996f8f18..0ff2e621 100644 --- a/images/shipyard/Dockerfile.ubuntu_xenial +++ b/images/shipyard/Dockerfile.ubuntu_xenial @@ -46,7 +46,6 @@ RUN set -ex && \ make \ netbase \ python3-dev \ - python3-setuptools \ --no-install-recommends \ && apt-get autoremove -yqq --purge \ && apt-get clean \ @@ -101,7 +100,8 @@ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/ ' \ && apt-get -qq update \ && apt-get -y install -y $buildDeps --no-install-recommends \ - && python3 -m pip install -U pip \ + && python3 -m pip install -U 'pip<21.0' \ + && python3 -m pip install -U setuptools \ && pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \ && cd /home/shipyard/shipyard_client \ && python3 setup.py install \ diff --git a/src/bin/shipyard_airflow/requirements.txt b/src/bin/shipyard_airflow/requirements.txt index 2a425f5b..788f3f64 100644 --- a/src/bin/shipyard_airflow/requirements.txt +++ b/src/bin/shipyard_airflow/requirements.txt @@ -58,3 +58,4 @@ tabulate==0.8.03 git+https://opendev.org/airship/deckhand.git@1f0c011a1708c1235a2be65edada7d386cd55d2a#egg=deckhand git+https://opendev.org/airship/drydock.git@96db568d4b28f5f163efbf606854243614a6c821#egg=drydock_provisioner&subdirectory=python git+https://opendev.org/airship/armada.git@0c3aff1ba1f0e836e3dd56de81b2ee560e11c179#egg=armada +git+https://opendev.org/airship/promenade.git@9cd261294bb77c42375ab02606c2f18bff081c26#egg=promenade diff --git a/src/bin/shipyard_airflow/test-requirements.txt b/src/bin/shipyard_airflow/test-requirements.txt index 7e5e50f1..726a8b2e 100644 --- a/src/bin/shipyard_airflow/test-requirements.txt +++ b/src/bin/shipyard_airflow/test-requirements.txt @@ -5,19 +5,15 @@ pytest==3.5.0 pytest-cov==2.5.1 responses==0.10.2 testfixtures==5.1.1 -# Had to upgrade apache-airflow as 1.10.1 requires a version of jinja too old -# for promenade -apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.2 -# install postgres seperately as apache-airflow 1.10.2 forces postgres to use +typing-extensions==3.7.2 +# Pin apache-airflow 1.10.5 to align with version in airflow images requirements.txt +apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5 +# install postgres seperately as apache-airflow 1.10.5 forces postgres to use # psycopg2 instead of psycopg2-binary which requires additional apt packages # to be installed, i.e. postgres-devel. Otherwise the following error is seen: # Error: pg_config executable not found. Only seems to be an issue in xenial postgres>=3.0.0 -# TODO(bryan-strassner) Pin to version for airflow when added to the -# requirements.txt in the airflow images directory -git+https://opendev.org/airship/promenade.git@master#egg=promenade - # Linting flake8<3.8.0,>=3.6.0 diff --git a/src/bin/shipyard_airflow/tests/unit/common/deployment_group/test_deployment_group_manager.py b/src/bin/shipyard_airflow/tests/unit/common/deployment_group/test_deployment_group_manager.py index 4c459b68..53a1c8d6 100644 --- a/src/bin/shipyard_airflow/tests/unit/common/deployment_group/test_deployment_group_manager.py +++ b/src/bin/shipyard_airflow/tests/unit/common/deployment_group/test_deployment_group_manager.py @@ -15,6 +15,7 @@ module """ import pytest +import re import yaml from shipyard_airflow.common.deployment_group.deployment_group import ( @@ -197,13 +198,14 @@ class TestDeploymentGroupManager: dgm.mark_group_prepared('compute-nodes-1') dgm.mark_group_failed('compute-nodes-2') dgm.report_group_summary() + ansi_re = re.compile(r'\x1b\[[0-9;]*m') assert "===== Group Summary =====" in caplog.text assert ("Group ntp-node [Critical] ended with stage: " - "Stage.NOT_STARTED") in caplog.text + "Stage.NOT_STARTED") in re.sub(ansi_re, '', caplog.text) caplog.clear() dgm.report_node_summary() - assert "Nodes Stage.PREPARED: node2" in caplog.text - assert "Nodes Stage.FAILED: node3" in caplog.text + assert "Nodes Stage.PREPARED: node2" in re.sub(ansi_re, '', caplog.text) + assert "Nodes Stage.FAILED: node3" in re.sub(ansi_re, '', caplog.text) assert "===== End Node Summary =====" in caplog.text assert "It was the best of times" not in caplog.text diff --git a/src/bin/shipyard_airflow/tox.ini b/src/bin/shipyard_airflow/tox.ini index 537cb57f..5c981018 100644 --- a/src/bin/shipyard_airflow/tox.ini +++ b/src/bin/shipyard_airflow/tox.ini @@ -1,17 +1,21 @@ [tox] +skipsdist=True envlist = py35, py36, pep8, bandit [testenv] setenv= PYTHONWARNING=all + LANG=C.UTF-8 + LC_ALL=C.UTF-8 deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt [testenv:py35] skipsdist=True setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes commands = + pip install -r{toxinidir}/test-requirements.txt + airflow initdb pytest \ {posargs} \ --cov-branch \ @@ -25,6 +29,8 @@ skipsdist=True setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes commands = + pip install -r{toxinidir}/test-requirements.txt + airflow initdb pytest \ {posargs} \ --cov-branch \ @@ -64,6 +70,7 @@ basepython=python3 setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes commands = + pip install -r{toxinidir}/test-requirements.txt oslo-config-generator --config-file=generator/config-generator.conf oslopolicy-sample-generator --config-file=generator/policy-generator.conf diff --git a/src/bin/shipyard_client/tox.ini b/src/bin/shipyard_client/tox.ini index 6a030ffa..256f569e 100644 --- a/src/bin/shipyard_client/tox.ini +++ b/src/bin/shipyard_client/tox.ini @@ -5,6 +5,8 @@ envlist = py35, py36, pep8, bandit [testenv] setenv= PYTHONWARNING=all + LC_ALL=C.UTF-8 + LC_ALL=C.UTF-8 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml index cb89f8ed..ab1b11a6 100644 --- a/tools/gate/playbooks/airskiff-deploy.yaml +++ b/tools/gate/playbooks/airskiff-deploy.yaml @@ -33,6 +33,7 @@ - name: Clone Required Repositories shell: | export CLONE_SHIPYARD={{ CLONE_SHIPYARD }} + export OSH_INFRA_COMMIT={{ HTK_COMMIT }} ./tools/deployment/airskiff/developer/000-clone-dependencies.sh args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" @@ -56,13 +57,6 @@ chdir: "{{ zuul.project.src_dir }}" become: yes - - name: Setup OpenStack Client - shell: | - ./tools/deployment/airskiff/developer/020-setup-client.sh - args: - chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" - become: yes - - name: Deploy Airship components using Armada shell: | mkdir ~/.kube diff --git a/tools/gate/playbooks/airskiff-reduce-site.yaml b/tools/gate/playbooks/airskiff-reduce-site.yaml index c0953a38..55f56ffe 100644 --- a/tools/gate/playbooks/airskiff-reduce-site.yaml +++ b/tools/gate/playbooks/airskiff-reduce-site.yaml @@ -14,9 +14,23 @@ - hosts: primary tasks: + - name: ensure pip packages are installed + apt: + name: "{{ item }}" + with_items: + - python3-pip + - python3-git + - python3-requests + - python3-yaml + become: yes + - name: Overwrite Armada manifest shell: | + git checkout v1.9 mv tools/gate/manifests/full-site.yaml \ type/skiff/manifests/full-site.yaml + # Overwrite versions.yaml with the latest Airship charts and xenial images + ./tools/updater.py --in-file global/software/config/versions.yaml \ + --tag-filter ubuntu_xenial --skip mariadb args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"