From 0e9d828fe61dd5f712b9701905421c0cde35d092 Mon Sep 17 00:00:00 2001 From: "SPEARS, DUSTIN (ds443n)" Date: Mon, 13 Feb 2023 10:11:12 -0500 Subject: [PATCH] Update armada to focal Update kubernetes client to v26.1.0 Updating armada to focal base image Remove xenial and opensuse dockerfiles Update tox python from py35 to py38 Add apparmor for docker build Uplift HTK chart version 0.2.52 Bumping up some python dependencies to get in sync with shipyard Added clear-firewall role for airskiff-deploy playbook Change-Id: If06a3f60466702d05a21c24a7cb8041bed41507a --- .zuul.yaml | 125 +++++++----------- Makefile | 4 +- armada/exceptions/base_exception.py | 10 +- armada/tests/unit/utils/test_source.py | 9 +- armada/utils/source.py | 6 +- armada/utils/validate.py | 2 +- images/armada/Dockerfile.opensuse_15 | 85 ------------ ....ubuntu_xenial => Dockerfile.ubuntu_focal} | 4 +- requirements.txt | 6 +- setup.cfg | 11 +- tools/gate/playbooks/airskiff-deploy.yaml | 11 ++ tools/gate/playbooks/docker-image-build.yaml | 4 + tools/helm_tk.sh | 2 +- tox.ini | 10 +- 14 files changed, 104 insertions(+), 185 deletions(-) delete mode 100644 images/armada/Dockerfile.opensuse_15 rename images/armada/{Dockerfile.ubuntu_xenial => Dockerfile.ubuntu_focal} (97%) diff --git a/.zuul.yaml b/.zuul.yaml index 2b4c27ec..90c64141 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -13,30 +13,28 @@ - project: templates: - openstack-cover-jobs - - openstack-python36-jobs + - openstack-python38-jobs check: jobs: - openstack-tox-docs - openstack-tox-pep8 - armada-chart-build-gate - armada-chart-build-latest-htk - - armada-docker-build-gate-ubuntu_bionic - - armada-docker-build-gate-ubuntu_xenial - - armada-docker-build-gate-opensuse + # - armada-docker-build-gate-ubuntu_bionic + - armada-docker-build-gate-ubuntu_focal - armada-airskiff-deploy gate: jobs: - openstack-tox-docs - openstack-tox-pep8 - armada-chart-build-gate - - armada-docker-build-gate-ubuntu_bionic - - armada-docker-build-gate-ubuntu_xenial - - armada-docker-build-gate-opensuse + # - armada-docker-build-gate-ubuntu_bionic + - armada-docker-build-gate-ubuntu_focal + - armada-airskiff-deploy post: jobs: - - armada-docker-publish-ubuntu_bionic - - armada-docker-publish-ubuntu_xenial - - armada-docker-publish-opensuse + - armada-docker-build-gate-ubuntu_focal + - armada-docker-publish-ubuntu_focal - armada-upload-git-mirror promote: jobs: @@ -48,13 +46,19 @@ - name: primary label: ubuntu-bionic +- nodeset: + name: armada-single-node-focal + nodes: + - name: primary + label: ubuntu-focal + - job: name: armada-chart-build-gate description: | Builds charts using pinned Helm toolkit. timeout: 900 run: tools/gate/playbooks/build-charts.yaml - nodeset: armada-single-node + nodeset: armada-single-node-focal - job: name: armada-chart-build-latest-htk @@ -63,49 +67,37 @@ timeout: 900 voting: false run: tools/gate/playbooks/build-charts.yaml - nodeset: armada-single-node + nodeset: armada-single-node-focal vars: HTK_COMMIT: master -- job: - name: armada-docker-build-gate-ubuntu_bionic - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node - vars: - publish: false - distro: ubuntu_bionic - tags: - dynamic: - patch_set: true +# - job: +# name: armada-docker-build-gate-ubuntu_bionic +# timeout: 1800 +# run: tools/gate/playbooks/docker-image-build.yaml +# nodeset: armada-single-node +# vars: +# publish: false +# distro: ubuntu_bionic +# tags: +# dynamic: +# patch_set: true - job: - name: armada-docker-build-gate-ubuntu_xenial + name: armada-docker-build-gate-ubuntu_focal timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node + nodeset: armada-single-node-focal vars: publish: false - distro: ubuntu_xenial - tags: - dynamic: - patch_set: true - -- job: - name: armada-docker-build-gate-opensuse - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node - vars: - publish: false - distro: opensuse_15 + distro: ubuntu_focal tags: dynamic: patch_set: true - job: name: armada-airskiff-deploy - nodeset: armada-single-node + nodeset: armada-single-node-focal description: | Deploy Memcached using Airskiff and submitted Armada changes. timeout: 9600 @@ -127,50 +119,33 @@ - ^releasenotes/.*$ - ^swagger/.*$ -- job: - name: armada-docker-publish-ubuntu_bionic - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node - secrets: - - airship_armada_quay_creds - vars: - publish: true - distro: ubuntu_bionic - tags: - dynamic: - branch: true - commit: true - static: - - latest +# - job: +# name: armada-docker-publish-ubuntu_bionic +# timeout: 1800 +# run: tools/gate/playbooks/docker-image-build.yaml +# nodeset: armada-single-node +# secrets: +# - airship_armada_quay_creds +# vars: +# publish: true +# distro: ubuntu_bionic +# tags: +# dynamic: +# branch: true +# commit: true +# static: +# - latest - job: - name: armada-docker-publish-ubuntu_xenial + name: armada-docker-publish-ubuntu_focal timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node + nodeset: armada-single-node-focal secrets: - airship_armada_quay_creds vars: publish: true - distro: ubuntu_xenial - tags: - dynamic: - branch: true - commit: true - static: - - latest - -- job: - name: armada-docker-publish-opensuse - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node - secrets: - - airship_armada_quay_creds - vars: - publish: true - distro: opensuse_15 + distro: ubuntu_focal tags: dynamic: branch: true diff --git a/Makefile b/Makefile index 8739aaab..b7918df3 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ LABEL ?= org.airshipit.build=community COMMIT ?= $(shell git rev-parse HEAD) PYTHON = python3 CHARTS := $(filter-out deps, $(patsubst charts/%/.,%,$(wildcard charts/*/.))) -DISTRO ?= ubuntu_bionic +DISTRO ?= ubuntu_focal IMAGE := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} UBUNTU_BASE_IMAGE ?= @@ -135,7 +135,7 @@ test-all: check-tox helm_lint tox test-unit: check-tox - tox -e py35 + tox -e py38 test-coverage: check-tox tox -e cover diff --git a/armada/exceptions/base_exception.py b/armada/exceptions/base_exception.py index 1a9fd3e9..366014da 100644 --- a/armada/exceptions/base_exception.py +++ b/armada/exceptions/base_exception.py @@ -14,6 +14,7 @@ import json import traceback +from contextlib import suppress import falcon from oslo_config import cfg @@ -251,10 +252,13 @@ class ArmadaBaseException(Exception): def __init__(self, message=None, **kwargs): self.message = message or self.message - try: # nosec + # replacing try-except-pass block with suppress + with suppress(Exception): self.message = self.message % kwargs - except Exception: - pass + # try: # nosec + # self.message = self.message % kwargs + # except Exception: + # pass super(ArmadaBaseException, self).__init__(self.message) diff --git a/armada/tests/unit/utils/test_source.py b/armada/tests/unit/utils/test_source.py index 2dd9245e..a966c670 100644 --- a/armada/tests/unit/utils/test_source.py +++ b/armada/tests/unit/utils/test_source.py @@ -110,7 +110,7 @@ class GitTestCase(base.ArmadaTestCase): source.download_tarball(url) mock_temp.mkstemp.assert_called_once() - mock_requests.get.assert_called_once_with(url, verify=False) + mock_requests.get.assert_called_once_with(url, timeout=5, verify=False) mock_open.assert_called_once_with('/tmp/armada', 'wb') mock_open().write.assert_called_once_with( mock_requests.get(url).content) @@ -121,7 +121,8 @@ class GitTestCase(base.ArmadaTestCase): def test_tarball_extract(self, mock_tarfile, mock_path, mock_temp): mock_path.exists.return_value = True mock_temp.mkdtemp.return_value = '/tmp/armada' - mock_opened_file = mock.Mock() + mock_opened_file = mock.MagicMock() + mock_opened_file.__iter__.return_value = ['file'] mock_tarfile.open.return_value = mock_opened_file path = '/tmp/mariadb-0.1.0.tgz' @@ -130,7 +131,7 @@ class GitTestCase(base.ArmadaTestCase): mock_path.exists.assert_called_once() mock_temp.mkdtemp.assert_called_once() mock_tarfile.open.assert_called_once_with(path) - mock_opened_file.extractall.assert_called_once_with('/tmp/armada') + mock_opened_file.extract.assert_called_once_with('file', '/tmp/armada') @test_utils.attr(type=['negative']) @mock.patch('armada.utils.source.os.path') @@ -144,7 +145,7 @@ class GitTestCase(base.ArmadaTestCase): path) mock_tarfile.open.assert_not_called() - mock_tarfile.extractall.assert_not_called() + mock_tarfile.extract.assert_not_called() @testtools.skipUnless( base.is_connected(), 'git clone requires network connectivity.') diff --git a/armada/utils/source.py b/armada/utils/source.py index ed42215b..1a471c1e 100644 --- a/armada/utils/source.py +++ b/armada/utils/source.py @@ -134,7 +134,8 @@ def download_tarball(tarball_url, verify=False, proxy_server=None): 'ftp': proxy_server } tarball_filename = tempfile.mkstemp(prefix='armada')[1] - response = requests.get(tarball_url, verify=verify, **kwargs) + response = requests.get( + tarball_url, timeout=5, verify=verify, **kwargs) with open(tarball_filename, 'wb') as f: f.write(response.content) @@ -155,7 +156,8 @@ def extract_tarball(tarball_path): try: file = tarfile.open(tarball_path) - file.extractall(temp_dir) + for member in file: + file.extract(member, temp_dir) except Exception: raise source_exceptions.TarballExtractException(tarball_path) return temp_dir diff --git a/armada/utils/validate.py b/armada/utils/validate.py index 043ef0a3..b4c46a49 100644 --- a/armada/utils/validate.py +++ b/armada/utils/validate.py @@ -177,6 +177,6 @@ def validate_armada_documents(documents): def validate_manifest_url(value): try: - return (requests.get(value).status_code == 200) + return (requests.get(value, timeout=5).status_code == 200) except requests.exceptions.RequestException: return False diff --git a/images/armada/Dockerfile.opensuse_15 b/images/armada/Dockerfile.opensuse_15 deleted file mode 100644 index bdb16939..00000000 --- a/images/armada/Dockerfile.opensuse_15 +++ /dev/null @@ -1,85 +0,0 @@ -ARG FROM=opensuse/leap:15.1 -FROM ${FROM} - -LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ - org.opencontainers.image.url='https://airshipit.org' \ - org.opencontainers.image.documentation='https://docs.airshipit.org/armada' \ - org.opencontainers.image.source='https://opendev.org/airship/armada' \ - org.opencontainers.image.vendor='The Airship Authors' \ - org.opencontainers.image.licenses='Apache-2.0' - -ENV LANG=C.UTF-8 -ENV LC_ALL=C.UTF-8 - -EXPOSE 8000 - -ENTRYPOINT ["./entrypoint.sh"] -CMD ["server"] - -RUN mkdir -p /armada && \ - zypper refresh && \ - zypper up -y && \ - zypper --non-interactive install \ - ca-certificates \ - curl \ - tar \ - gzip \ - netcfg \ - python3 \ - python3-devel \ - python3-setuptools \ - python3-pip \ - automake \ - gcc \ - git \ - libopenssl-devel \ - libtool \ - make - -RUN pip3 install --upgrade pip && \ - zypper clean -a && \ - rm -rf \ - /tmp/* \ - /var/tmp/* \ - /usr/share/man \ - /usr/share/doc \ - /usr/share/doc-base - -ARG HELM_ARTIFACT_URL -RUN \ - curl -fSSL -O ${HELM_ARTIFACT_URL} && \ - tar -xvf $(basename ${HELM_ARTIFACT_URL}) && \ - mv linux-amd64/helm /usr/local/bin - -ENV LD_LIBRARY_PATH=/usr/local/lib - -ARG LIBYAML_VERSION=0.2.5 -RUN set -ex \ - && git clone https://github.com/yaml/libyaml.git \ - && cd libyaml \ - && git checkout $LIBYAML_VERSION \ - && ./bootstrap \ - && ./configure \ - && make \ - && make install \ - && cd .. \ - && rm -fr libyaml - -WORKDIR /armada - -COPY requirements.txt /tmp/ - -RUN \ - pip3 install -r /tmp/requirements.txt && \ - useradd -u 1000 -g users -d /armada armada && \ - rm -rf /tmp/requirements.txt - -COPY . /armada - -RUN \ - mv etc/armada /etc/ && \ - cd /armada && \ - chown -R armada:users /armada && \ - python3 setup.py install - -USER armada diff --git a/images/armada/Dockerfile.ubuntu_xenial b/images/armada/Dockerfile.ubuntu_focal similarity index 97% rename from images/armada/Dockerfile.ubuntu_xenial rename to images/armada/Dockerfile.ubuntu_focal index 71fc946e..59513e2e 100644 --- a/images/armada/Dockerfile.ubuntu_xenial +++ b/images/armada/Dockerfile.ubuntu_focal @@ -1,4 +1,4 @@ -ARG FROM=ubuntu:16.04 +ARG FROM=ubuntu:20.04 FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ @@ -70,7 +70,7 @@ RUN set -ex \ && make install \ && cd .. \ && rm -fr libyaml \ - && python3 -m pip install -U 'pip<21.0' \ + && python3 -m pip install -U pip \ && pip3 install -r requirements.txt --no-cache-dir \ && curl -fSSL -O ${HELM_ARTIFACT_URL} \ && tar -xvf $(basename ${HELM_ARTIFACT_URL}) \ diff --git a/requirements.txt b/requirements.txt index 6246d4ce..89477954 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -amqp<2.7,>=2.6.0 +amqp==5.0.8 deepdiff>=3.3.0 gitpython jsonschema>=3.0.1,<4 keystoneauth1>=3.18.0 keystonemiddleware>=5.3.0 -kombu<4.7,>=4.6.10 -kubernetes~=25.3.0; python_version >= '3.6' +kombu==5.1.0 +kubernetes~=26.1.0; python_version >= '3.6' Paste>=2.0.3 PasteDeploy>=1.5.2 pylibyaml~=0.1 diff --git a/setup.cfg b/setup.cfg index 8d4a852b..5a110780 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,11 @@ [metadata] name = Armada +version = 1.0 summary = Tool for managing multiple Helm charts with dependencies by centralizing all configurations in a single Armada YAML and providing life-cycle hooks for all Helm releases. -description-file = README.rst +description_file = README.rst author = The Airship Authors -author-email = airship-discuss@lists.airshipit.org -home-page = https://airshipit.org +author_email = airship-discuss@lists.airshipit.org +home_page = https://airshipit.org classifier = Intended Audience :: Information Technology @@ -13,8 +14,8 @@ classifier = Operating System :: POSIX :: Linux Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.10 [files] packages = diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml index f654301d..c436cd77 100644 --- a/tools/gate/playbooks/airskiff-deploy.yaml +++ b/tools/gate/playbooks/airskiff-deploy.yaml @@ -13,6 +13,17 @@ # limitations under the License. - hosts: primary + vars: + env: + HTTP_PROXY: "" + HTTPS_PROXY: "" + NO_PROXY: "" + DISTRO: "{{ distro }}" + become: true + + roles: + - clear-firewall + tasks: # Stop systemd-resolved service before starting docker. - name: stop systemd-resolved service diff --git a/tools/gate/playbooks/docker-image-build.yaml b/tools/gate/playbooks/docker-image-build.yaml index a8effedd..08d450e5 100644 --- a/tools/gate/playbooks/docker-image-build.yaml +++ b/tools/gate/playbooks/docker-image-build.yaml @@ -72,7 +72,11 @@ name: "{{ item }}" allow_unauthenticated: True with_items: + - apparmor - docker-ce + - docker-ce-cli + - docker-buildx-plugin + - containerd.io - python3-pip - python3-setuptools - pip: diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh index 34a0aa9d..fdac0bca 100755 --- a/tools/helm_tk.sh +++ b/tools/helm_tk.sh @@ -17,7 +17,7 @@ set -eux HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"} -HTK_STABLE_COMMIT=${HTK_COMMIT:-"f4972121bcb41c8d74748917804d2b239ab757f9"} +HTK_STABLE_COMMIT=${HTK_COMMIT:-"a7cd689280cdbc0acd04a7a1b745941260e8700b"} TMP_DIR=$(mktemp -d) diff --git a/tox.ini b/tox.ini index f625eb16..623f2d6b 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,13 @@ envlist = py38, pep8, cover, bandit deps= -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -passenv=HTTP_PROXY,HTTPS_PROXY,http_proxy,https_proxy,NO_PROXY,no_proxy +passenv= + HTTP_PROXY + HTTPS_PROXY + http_proxy + https_proxy + NO_PROXY + no_proxy setenv= VIRTUAL_ENV={envdir} usedevelop = True @@ -101,7 +107,7 @@ show-source = true # [H904] Delay string interpolations at logging calls. enable-extensions = H106,H201,H904 # [W503] line break before binary operator -ignore = W503 +ignore = W503,B202,B113 exclude = .git,.tox,dist,*lib/python*,*egg,build,releasenotes,doc/*,venv max-complexity = 24 application-import-names = armada