diff --git a/.zuul.yaml b/.zuul.yaml index 2b4c27ec..90c64141 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -13,30 +13,28 @@ - project: templates: - openstack-cover-jobs - - openstack-python36-jobs + - openstack-python38-jobs check: jobs: - openstack-tox-docs - openstack-tox-pep8 - armada-chart-build-gate - armada-chart-build-latest-htk - - armada-docker-build-gate-ubuntu_bionic - - armada-docker-build-gate-ubuntu_xenial - - armada-docker-build-gate-opensuse + # - armada-docker-build-gate-ubuntu_bionic + - armada-docker-build-gate-ubuntu_focal - armada-airskiff-deploy gate: jobs: - openstack-tox-docs - openstack-tox-pep8 - armada-chart-build-gate - - armada-docker-build-gate-ubuntu_bionic - - armada-docker-build-gate-ubuntu_xenial - - armada-docker-build-gate-opensuse + # - armada-docker-build-gate-ubuntu_bionic + - armada-docker-build-gate-ubuntu_focal + - armada-airskiff-deploy post: jobs: - - armada-docker-publish-ubuntu_bionic - - armada-docker-publish-ubuntu_xenial - - armada-docker-publish-opensuse + - armada-docker-build-gate-ubuntu_focal + - armada-docker-publish-ubuntu_focal - armada-upload-git-mirror promote: jobs: @@ -48,13 +46,19 @@ - name: primary label: ubuntu-bionic +- nodeset: + name: armada-single-node-focal + nodes: + - name: primary + label: ubuntu-focal + - job: name: armada-chart-build-gate description: | Builds charts using pinned Helm toolkit. timeout: 900 run: tools/gate/playbooks/build-charts.yaml - nodeset: armada-single-node + nodeset: armada-single-node-focal - job: name: armada-chart-build-latest-htk @@ -63,49 +67,37 @@ timeout: 900 voting: false run: tools/gate/playbooks/build-charts.yaml - nodeset: armada-single-node + nodeset: armada-single-node-focal vars: HTK_COMMIT: master -- job: - name: armada-docker-build-gate-ubuntu_bionic - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node - vars: - publish: false - distro: ubuntu_bionic - tags: - dynamic: - patch_set: true +# - job: +# name: armada-docker-build-gate-ubuntu_bionic +# timeout: 1800 +# run: tools/gate/playbooks/docker-image-build.yaml +# nodeset: armada-single-node +# vars: +# publish: false +# distro: ubuntu_bionic +# tags: +# dynamic: +# patch_set: true - job: - name: armada-docker-build-gate-ubuntu_xenial + name: armada-docker-build-gate-ubuntu_focal timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node + nodeset: armada-single-node-focal vars: publish: false - distro: ubuntu_xenial - tags: - dynamic: - patch_set: true - -- job: - name: armada-docker-build-gate-opensuse - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node - vars: - publish: false - distro: opensuse_15 + distro: ubuntu_focal tags: dynamic: patch_set: true - job: name: armada-airskiff-deploy - nodeset: armada-single-node + nodeset: armada-single-node-focal description: | Deploy Memcached using Airskiff and submitted Armada changes. timeout: 9600 @@ -127,50 +119,33 @@ - ^releasenotes/.*$ - ^swagger/.*$ -- job: - name: armada-docker-publish-ubuntu_bionic - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node - secrets: - - airship_armada_quay_creds - vars: - publish: true - distro: ubuntu_bionic - tags: - dynamic: - branch: true - commit: true - static: - - latest +# - job: +# name: armada-docker-publish-ubuntu_bionic +# timeout: 1800 +# run: tools/gate/playbooks/docker-image-build.yaml +# nodeset: armada-single-node +# secrets: +# - airship_armada_quay_creds +# vars: +# publish: true +# distro: ubuntu_bionic +# tags: +# dynamic: +# branch: true +# commit: true +# static: +# - latest - job: - name: armada-docker-publish-ubuntu_xenial + name: armada-docker-publish-ubuntu_focal timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node + nodeset: armada-single-node-focal secrets: - airship_armada_quay_creds vars: publish: true - distro: ubuntu_xenial - tags: - dynamic: - branch: true - commit: true - static: - - latest - -- job: - name: armada-docker-publish-opensuse - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node - secrets: - - airship_armada_quay_creds - vars: - publish: true - distro: opensuse_15 + distro: ubuntu_focal tags: dynamic: branch: true diff --git a/Makefile b/Makefile index 8739aaab..b7918df3 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ LABEL ?= org.airshipit.build=community COMMIT ?= $(shell git rev-parse HEAD) PYTHON = python3 CHARTS := $(filter-out deps, $(patsubst charts/%/.,%,$(wildcard charts/*/.))) -DISTRO ?= ubuntu_bionic +DISTRO ?= ubuntu_focal IMAGE := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} UBUNTU_BASE_IMAGE ?= @@ -135,7 +135,7 @@ test-all: check-tox helm_lint tox test-unit: check-tox - tox -e py35 + tox -e py38 test-coverage: check-tox tox -e cover diff --git a/armada/exceptions/base_exception.py b/armada/exceptions/base_exception.py index 1a9fd3e9..366014da 100644 --- a/armada/exceptions/base_exception.py +++ b/armada/exceptions/base_exception.py @@ -14,6 +14,7 @@ import json import traceback +from contextlib import suppress import falcon from oslo_config import cfg @@ -251,10 +252,13 @@ class ArmadaBaseException(Exception): def __init__(self, message=None, **kwargs): self.message = message or self.message - try: # nosec + # replacing try-except-pass block with suppress + with suppress(Exception): self.message = self.message % kwargs - except Exception: - pass + # try: # nosec + # self.message = self.message % kwargs + # except Exception: + # pass super(ArmadaBaseException, self).__init__(self.message) diff --git a/armada/tests/unit/utils/test_source.py b/armada/tests/unit/utils/test_source.py index 2dd9245e..a966c670 100644 --- a/armada/tests/unit/utils/test_source.py +++ b/armada/tests/unit/utils/test_source.py @@ -110,7 +110,7 @@ class GitTestCase(base.ArmadaTestCase): source.download_tarball(url) mock_temp.mkstemp.assert_called_once() - mock_requests.get.assert_called_once_with(url, verify=False) + mock_requests.get.assert_called_once_with(url, timeout=5, verify=False) mock_open.assert_called_once_with('/tmp/armada', 'wb') mock_open().write.assert_called_once_with( mock_requests.get(url).content) @@ -121,7 +121,8 @@ class GitTestCase(base.ArmadaTestCase): def test_tarball_extract(self, mock_tarfile, mock_path, mock_temp): mock_path.exists.return_value = True mock_temp.mkdtemp.return_value = '/tmp/armada' - mock_opened_file = mock.Mock() + mock_opened_file = mock.MagicMock() + mock_opened_file.__iter__.return_value = ['file'] mock_tarfile.open.return_value = mock_opened_file path = '/tmp/mariadb-0.1.0.tgz' @@ -130,7 +131,7 @@ class GitTestCase(base.ArmadaTestCase): mock_path.exists.assert_called_once() mock_temp.mkdtemp.assert_called_once() mock_tarfile.open.assert_called_once_with(path) - mock_opened_file.extractall.assert_called_once_with('/tmp/armada') + mock_opened_file.extract.assert_called_once_with('file', '/tmp/armada') @test_utils.attr(type=['negative']) @mock.patch('armada.utils.source.os.path') @@ -144,7 +145,7 @@ class GitTestCase(base.ArmadaTestCase): path) mock_tarfile.open.assert_not_called() - mock_tarfile.extractall.assert_not_called() + mock_tarfile.extract.assert_not_called() @testtools.skipUnless( base.is_connected(), 'git clone requires network connectivity.') diff --git a/armada/utils/source.py b/armada/utils/source.py index ed42215b..1a471c1e 100644 --- a/armada/utils/source.py +++ b/armada/utils/source.py @@ -134,7 +134,8 @@ def download_tarball(tarball_url, verify=False, proxy_server=None): 'ftp': proxy_server } tarball_filename = tempfile.mkstemp(prefix='armada')[1] - response = requests.get(tarball_url, verify=verify, **kwargs) + response = requests.get( + tarball_url, timeout=5, verify=verify, **kwargs) with open(tarball_filename, 'wb') as f: f.write(response.content) @@ -155,7 +156,8 @@ def extract_tarball(tarball_path): try: file = tarfile.open(tarball_path) - file.extractall(temp_dir) + for member in file: + file.extract(member, temp_dir) except Exception: raise source_exceptions.TarballExtractException(tarball_path) return temp_dir diff --git a/armada/utils/validate.py b/armada/utils/validate.py index 043ef0a3..b4c46a49 100644 --- a/armada/utils/validate.py +++ b/armada/utils/validate.py @@ -177,6 +177,6 @@ def validate_armada_documents(documents): def validate_manifest_url(value): try: - return (requests.get(value).status_code == 200) + return (requests.get(value, timeout=5).status_code == 200) except requests.exceptions.RequestException: return False diff --git a/images/armada/Dockerfile.opensuse_15 b/images/armada/Dockerfile.opensuse_15 deleted file mode 100644 index bdb16939..00000000 --- a/images/armada/Dockerfile.opensuse_15 +++ /dev/null @@ -1,85 +0,0 @@ -ARG FROM=opensuse/leap:15.1 -FROM ${FROM} - -LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ - org.opencontainers.image.url='https://airshipit.org' \ - org.opencontainers.image.documentation='https://docs.airshipit.org/armada' \ - org.opencontainers.image.source='https://opendev.org/airship/armada' \ - org.opencontainers.image.vendor='The Airship Authors' \ - org.opencontainers.image.licenses='Apache-2.0' - -ENV LANG=C.UTF-8 -ENV LC_ALL=C.UTF-8 - -EXPOSE 8000 - -ENTRYPOINT ["./entrypoint.sh"] -CMD ["server"] - -RUN mkdir -p /armada && \ - zypper refresh && \ - zypper up -y && \ - zypper --non-interactive install \ - ca-certificates \ - curl \ - tar \ - gzip \ - netcfg \ - python3 \ - python3-devel \ - python3-setuptools \ - python3-pip \ - automake \ - gcc \ - git \ - libopenssl-devel \ - libtool \ - make - -RUN pip3 install --upgrade pip && \ - zypper clean -a && \ - rm -rf \ - /tmp/* \ - /var/tmp/* \ - /usr/share/man \ - /usr/share/doc \ - /usr/share/doc-base - -ARG HELM_ARTIFACT_URL -RUN \ - curl -fSSL -O ${HELM_ARTIFACT_URL} && \ - tar -xvf $(basename ${HELM_ARTIFACT_URL}) && \ - mv linux-amd64/helm /usr/local/bin - -ENV LD_LIBRARY_PATH=/usr/local/lib - -ARG LIBYAML_VERSION=0.2.5 -RUN set -ex \ - && git clone https://github.com/yaml/libyaml.git \ - && cd libyaml \ - && git checkout $LIBYAML_VERSION \ - && ./bootstrap \ - && ./configure \ - && make \ - && make install \ - && cd .. \ - && rm -fr libyaml - -WORKDIR /armada - -COPY requirements.txt /tmp/ - -RUN \ - pip3 install -r /tmp/requirements.txt && \ - useradd -u 1000 -g users -d /armada armada && \ - rm -rf /tmp/requirements.txt - -COPY . /armada - -RUN \ - mv etc/armada /etc/ && \ - cd /armada && \ - chown -R armada:users /armada && \ - python3 setup.py install - -USER armada diff --git a/images/armada/Dockerfile.ubuntu_xenial b/images/armada/Dockerfile.ubuntu_focal similarity index 97% rename from images/armada/Dockerfile.ubuntu_xenial rename to images/armada/Dockerfile.ubuntu_focal index 71fc946e..59513e2e 100644 --- a/images/armada/Dockerfile.ubuntu_xenial +++ b/images/armada/Dockerfile.ubuntu_focal @@ -1,4 +1,4 @@ -ARG FROM=ubuntu:16.04 +ARG FROM=ubuntu:20.04 FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ @@ -70,7 +70,7 @@ RUN set -ex \ && make install \ && cd .. \ && rm -fr libyaml \ - && python3 -m pip install -U 'pip<21.0' \ + && python3 -m pip install -U pip \ && pip3 install -r requirements.txt --no-cache-dir \ && curl -fSSL -O ${HELM_ARTIFACT_URL} \ && tar -xvf $(basename ${HELM_ARTIFACT_URL}) \ diff --git a/requirements.txt b/requirements.txt index 6246d4ce..89477954 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -amqp<2.7,>=2.6.0 +amqp==5.0.8 deepdiff>=3.3.0 gitpython jsonschema>=3.0.1,<4 keystoneauth1>=3.18.0 keystonemiddleware>=5.3.0 -kombu<4.7,>=4.6.10 -kubernetes~=25.3.0; python_version >= '3.6' +kombu==5.1.0 +kubernetes~=26.1.0; python_version >= '3.6' Paste>=2.0.3 PasteDeploy>=1.5.2 pylibyaml~=0.1 diff --git a/setup.cfg b/setup.cfg index 8d4a852b..5a110780 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,11 @@ [metadata] name = Armada +version = 1.0 summary = Tool for managing multiple Helm charts with dependencies by centralizing all configurations in a single Armada YAML and providing life-cycle hooks for all Helm releases. -description-file = README.rst +description_file = README.rst author = The Airship Authors -author-email = airship-discuss@lists.airshipit.org -home-page = https://airshipit.org +author_email = airship-discuss@lists.airshipit.org +home_page = https://airshipit.org classifier = Intended Audience :: Information Technology @@ -13,8 +14,8 @@ classifier = Operating System :: POSIX :: Linux Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.10 [files] packages = diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml index f654301d..c436cd77 100644 --- a/tools/gate/playbooks/airskiff-deploy.yaml +++ b/tools/gate/playbooks/airskiff-deploy.yaml @@ -13,6 +13,17 @@ # limitations under the License. - hosts: primary + vars: + env: + HTTP_PROXY: "" + HTTPS_PROXY: "" + NO_PROXY: "" + DISTRO: "{{ distro }}" + become: true + + roles: + - clear-firewall + tasks: # Stop systemd-resolved service before starting docker. - name: stop systemd-resolved service diff --git a/tools/gate/playbooks/docker-image-build.yaml b/tools/gate/playbooks/docker-image-build.yaml index a8effedd..08d450e5 100644 --- a/tools/gate/playbooks/docker-image-build.yaml +++ b/tools/gate/playbooks/docker-image-build.yaml @@ -72,7 +72,11 @@ name: "{{ item }}" allow_unauthenticated: True with_items: + - apparmor - docker-ce + - docker-ce-cli + - docker-buildx-plugin + - containerd.io - python3-pip - python3-setuptools - pip: diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh index 34a0aa9d..fdac0bca 100755 --- a/tools/helm_tk.sh +++ b/tools/helm_tk.sh @@ -17,7 +17,7 @@ set -eux HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"} -HTK_STABLE_COMMIT=${HTK_COMMIT:-"f4972121bcb41c8d74748917804d2b239ab757f9"} +HTK_STABLE_COMMIT=${HTK_COMMIT:-"a7cd689280cdbc0acd04a7a1b745941260e8700b"} TMP_DIR=$(mktemp -d) diff --git a/tox.ini b/tox.ini index f625eb16..623f2d6b 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,13 @@ envlist = py38, pep8, cover, bandit deps= -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -passenv=HTTP_PROXY,HTTPS_PROXY,http_proxy,https_proxy,NO_PROXY,no_proxy +passenv= + HTTP_PROXY + HTTPS_PROXY + http_proxy + https_proxy + NO_PROXY + no_proxy setenv= VIRTUAL_ENV={envdir} usedevelop = True @@ -101,7 +107,7 @@ show-source = true # [H904] Delay string interpolations at logging calls. enable-extensions = H106,H201,H904 # [W503] line break before binary operator -ignore = W503 +ignore = W503,B202,B113 exclude = .git,.tox,dist,*lib/python*,*egg,build,releasenotes,doc/*,venv max-complexity = 24 application-import-names = armada