airship
/
armada
Code Issues Proposed changes

Update armada to focal 

Update kubernetes client to v26.1.0
Updating armada to focal base image
Remove xenial and opensuse dockerfiles
Update tox python from py35 to py38
Add apparmor for docker build
Uplift HTK chart version 0.2.52
Bumping up some python dependencies to get in sync with shipyard
Added clear-firewall role for airskiff-deploy playbook

Change-Id: If06a3f60466702d05a21c24a7cb8041bed41507a
This commit is contained in:
SPEARS, DUSTIN (ds443n) 2023-02-13 10:11:12 -05:00
parent 099de8aaf4
commit 0e9d828fe6
14 changed files with 104 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
.zuul.yaml
View File

@ -13,30 +13,28 @@
- project: - project:
templates: templates:
- openstack-cover-jobs - openstack-cover-jobs
- openstack-python36-jobs - openstack-python38-jobs
check: check:
jobs: jobs:
- openstack-tox-docs - openstack-tox-docs
- openstack-tox-pep8 - openstack-tox-pep8
- armada-chart-build-gate - armada-chart-build-gate
- armada-chart-build-latest-htk - armada-chart-build-latest-htk
- armada-docker-build-gate-ubuntu_bionic # - armada-docker-build-gate-ubuntu_bionic
- armada-docker-build-gate-ubuntu_xenial - armada-docker-build-gate-ubuntu_focal
- armada-docker-build-gate-opensuse
- armada-airskiff-deploy - armada-airskiff-deploy
gate: gate:
jobs: jobs:
- openstack-tox-docs - openstack-tox-docs
- openstack-tox-pep8 - openstack-tox-pep8
- armada-chart-build-gate - armada-chart-build-gate
- armada-docker-build-gate-ubuntu_bionic # - armada-docker-build-gate-ubuntu_bionic
- armada-docker-build-gate-ubuntu_xenial - armada-docker-build-gate-ubuntu_focal
- armada-docker-build-gate-opensuse - armada-airskiff-deploy
post: post:
jobs: jobs:
- armada-docker-publish-ubuntu_bionic - armada-docker-build-gate-ubuntu_focal
- armada-docker-publish-ubuntu_xenial - armada-docker-publish-ubuntu_focal
- armada-docker-publish-opensuse
- armada-upload-git-mirror - armada-upload-git-mirror
promote: promote:
jobs: jobs:
@ -48,13 +46,19 @@
- name: primary - name: primary
label: ubuntu-bionic label: ubuntu-bionic
- nodeset:
name: armada-single-node-focal
nodes:
- name: primary
label: ubuntu-focal
- job: - job:
name: armada-chart-build-gate name: armada-chart-build-gate
description: | description: |
Builds charts using pinned Helm toolkit. Builds charts using pinned Helm toolkit.
timeout: 900 timeout: 900
run: tools/gate/playbooks/build-charts.yaml run: tools/gate/playbooks/build-charts.yaml
nodeset: armada-single-node nodeset: armada-single-node-focal
- job: - job:
name: armada-chart-build-latest-htk name: armada-chart-build-latest-htk
@ -63,49 +67,37 @@
timeout: 900 timeout: 900
voting: false voting: false
run: tools/gate/playbooks/build-charts.yaml run: tools/gate/playbooks/build-charts.yaml
nodeset: armada-single-node nodeset: armada-single-node-focal
vars: vars:
HTK_COMMIT: master HTK_COMMIT: master
- job: # - job:
name: armada-docker-build-gate-ubuntu_bionic # name: armada-docker-build-gate-ubuntu_bionic
timeout: 1800 # timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml # run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node # nodeset: armada-single-node
vars: # vars:
publish: false # publish: false
distro: ubuntu_bionic # distro: ubuntu_bionic
tags: # tags:
dynamic: # dynamic:
patch_set: true # patch_set: true
- job: - job:
name: armada-docker-build-gate-ubuntu_xenial name: armada-docker-build-gate-ubuntu_focal
timeout: 1800 timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node nodeset: armada-single-node-focal
vars: vars:
publish: false publish: false
distro: ubuntu_xenial distro: ubuntu_focal
tags:
dynamic:
patch_set: true
- job:
name: armada-docker-build-gate-opensuse
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
vars:
publish: false
distro: opensuse_15
tags: tags:
dynamic: dynamic:
patch_set: true patch_set: true
- job: - job:
name: armada-airskiff-deploy name: armada-airskiff-deploy
nodeset: armada-single-node nodeset: armada-single-node-focal
description: | description: |
Deploy Memcached using Airskiff and submitted Armada changes. Deploy Memcached using Airskiff and submitted Armada changes.
timeout: 9600 timeout: 9600
@ -127,50 +119,33 @@
- ^releasenotes/.*$ - ^releasenotes/.*$
- ^swagger/.*$ - ^swagger/.*$
- job: # - job:
name: armada-docker-publish-ubuntu_bionic # name: armada-docker-publish-ubuntu_bionic
timeout: 1800 # timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml # run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node # nodeset: armada-single-node
secrets: # secrets:
- airship_armada_quay_creds # - airship_armada_quay_creds
vars: # vars:
publish: true # publish: true
distro: ubuntu_bionic # distro: ubuntu_bionic
tags: # tags:
dynamic: # dynamic:
branch: true # branch: true
commit: true # commit: true
static: # static:
- latest # - latest
- job: - job:
name: armada-docker-publish-ubuntu_xenial name: armada-docker-publish-ubuntu_focal
timeout: 1800 timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node nodeset: armada-single-node-focal
secrets: secrets:
- airship_armada_quay_creds - airship_armada_quay_creds
vars: vars:
publish: true publish: true
distro: ubuntu_xenial distro: ubuntu_focal
tags:
dynamic:
branch: true
commit: true
static:
- latest
- job:
name: armada-docker-publish-opensuse
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
secrets:
- airship_armada_quay_creds
vars:
publish: true
distro: opensuse_15
tags: tags:
dynamic: dynamic:
branch: true branch: true

4
Makefile
View File

@ -28,7 +28,7 @@ LABEL ?= org.airshipit.build=community
COMMIT ?= $(shell git rev-parse HEAD) COMMIT ?= $(shell git rev-parse HEAD)
PYTHON = python3 PYTHON = python3
CHARTS := $(filter-out deps, $(patsubst charts/%/.,%,$(wildcard charts/*/.))) CHARTS := $(filter-out deps, $(patsubst charts/%/.,%,$(wildcard charts/*/.)))
DISTRO ?= ubuntu_bionic DISTRO ?= ubuntu_focal
IMAGE := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} IMAGE := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
UBUNTU_BASE_IMAGE ?= UBUNTU_BASE_IMAGE ?=
@ -135,7 +135,7 @@ test-all: check-tox helm_lint
tox tox
test-unit: check-tox test-unit: check-tox
tox -e py35 tox -e py38
test-coverage: check-tox test-coverage: check-tox
tox -e cover tox -e cover

10
armada/exceptions/base_exception.py
View File

@ -14,6 +14,7 @@
import json import json
import traceback import traceback
from contextlib import suppress
import falcon import falcon
from oslo_config import cfg from oslo_config import cfg
@ -251,10 +252,13 @@ class ArmadaBaseException(Exception):
def __init__(self, message=None, **kwargs): def __init__(self, message=None, **kwargs):
self.message = message or self.message self.message = message or self.message
try: # nosec # replacing try-except-pass block with suppress
with suppress(Exception):
self.message = self.message % kwargs self.message = self.message % kwargs
except Exception: # try: # nosec
pass # self.message = self.message % kwargs
# except Exception:
# pass
super(ArmadaBaseException, self).__init__(self.message) super(ArmadaBaseException, self).__init__(self.message)

9
armada/tests/unit/utils/test_source.py
View File

@ -110,7 +110,7 @@ class GitTestCase(base.ArmadaTestCase):
source.download_tarball(url) source.download_tarball(url)
mock_temp.mkstemp.assert_called_once() mock_temp.mkstemp.assert_called_once()
mock_requests.get.assert_called_once_with(url, verify=False) mock_requests.get.assert_called_once_with(url, timeout=5, verify=False)
mock_open.assert_called_once_with('/tmp/armada', 'wb') mock_open.assert_called_once_with('/tmp/armada', 'wb')
mock_open().write.assert_called_once_with( mock_open().write.assert_called_once_with(
mock_requests.get(url).content) mock_requests.get(url).content)
@ -121,7 +121,8 @@ class GitTestCase(base.ArmadaTestCase):
def test_tarball_extract(self, mock_tarfile, mock_path, mock_temp): def test_tarball_extract(self, mock_tarfile, mock_path, mock_temp):
mock_path.exists.return_value = True mock_path.exists.return_value = True
mock_temp.mkdtemp.return_value = '/tmp/armada' mock_temp.mkdtemp.return_value = '/tmp/armada'
mock_opened_file = mock.Mock() mock_opened_file = mock.MagicMock()
mock_opened_file.__iter__.return_value = ['file']
mock_tarfile.open.return_value = mock_opened_file mock_tarfile.open.return_value = mock_opened_file
path = '/tmp/mariadb-0.1.0.tgz' path = '/tmp/mariadb-0.1.0.tgz'
@ -130,7 +131,7 @@ class GitTestCase(base.ArmadaTestCase):
mock_path.exists.assert_called_once() mock_path.exists.assert_called_once()
mock_temp.mkdtemp.assert_called_once() mock_temp.mkdtemp.assert_called_once()
mock_tarfile.open.assert_called_once_with(path) mock_tarfile.open.assert_called_once_with(path)
mock_opened_file.extractall.assert_called_once_with('/tmp/armada') mock_opened_file.extract.assert_called_once_with('file', '/tmp/armada')
@test_utils.attr(type=['negative']) @test_utils.attr(type=['negative'])
@mock.patch('armada.utils.source.os.path') @mock.patch('armada.utils.source.os.path')
@ -144,7 +145,7 @@ class GitTestCase(base.ArmadaTestCase):
path) path)
mock_tarfile.open.assert_not_called() mock_tarfile.open.assert_not_called()
mock_tarfile.extractall.assert_not_called() mock_tarfile.extract.assert_not_called()
@testtools.skipUnless( @testtools.skipUnless(
base.is_connected(), 'git clone requires network connectivity.') base.is_connected(), 'git clone requires network connectivity.')

6
armada/utils/source.py
View File

@ -134,7 +134,8 @@ def download_tarball(tarball_url, verify=False, proxy_server=None):
'ftp': proxy_server 'ftp': proxy_server
} }
tarball_filename = tempfile.mkstemp(prefix='armada')[1] tarball_filename = tempfile.mkstemp(prefix='armada')[1]
response = requests.get(tarball_url, verify=verify, **kwargs) response = requests.get(
tarball_url, timeout=5, verify=verify, **kwargs)
with open(tarball_filename, 'wb') as f: with open(tarball_filename, 'wb') as f:
f.write(response.content) f.write(response.content)
@ -155,7 +156,8 @@ def extract_tarball(tarball_path):
try: try:
file = tarfile.open(tarball_path) file = tarfile.open(tarball_path)
file.extractall(temp_dir) for member in file:
file.extract(member, temp_dir)
except Exception: except Exception:
raise source_exceptions.TarballExtractException(tarball_path) raise source_exceptions.TarballExtractException(tarball_path)
return temp_dir return temp_dir

2
armada/utils/validate.py
View File

@ -177,6 +177,6 @@ def validate_armada_documents(documents):
def validate_manifest_url(value): def validate_manifest_url(value):
try: try:
return (requests.get(value).status_code == 200) return (requests.get(value, timeout=5).status_code == 200)
except requests.exceptions.RequestException: except requests.exceptions.RequestException:
return False return False

85
images/armada/Dockerfile.opensuse_15
View File

@ -1,85 +0,0 @@
ARG FROM=opensuse/leap:15.1
FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
org.opencontainers.image.url='https://airshipit.org' \
org.opencontainers.image.documentation='https://docs.airshipit.org/armada' \
org.opencontainers.image.source='https://opendev.org/airship/armada' \
org.opencontainers.image.vendor='The Airship Authors' \
org.opencontainers.image.licenses='Apache-2.0'
ENV LANG=C.UTF-8
ENV LC_ALL=C.UTF-8
EXPOSE 8000
ENTRYPOINT ["./entrypoint.sh"]
CMD ["server"]
RUN mkdir -p /armada && \
zypper refresh && \
zypper up -y && \
zypper --non-interactive install \
ca-certificates \
curl \
tar \
gzip \
netcfg \
python3 \
python3-devel \
python3-setuptools \
python3-pip \
automake \
gcc \
git \
libopenssl-devel \
libtool \
make
RUN pip3 install --upgrade pip && \
zypper clean -a && \
rm -rf \
/tmp/* \
/var/tmp/* \
/usr/share/man \
/usr/share/doc \
/usr/share/doc-base
ARG HELM_ARTIFACT_URL
RUN \
curl -fSSL -O ${HELM_ARTIFACT_URL} && \
tar -xvf $(basename ${HELM_ARTIFACT_URL}) && \
mv linux-amd64/helm /usr/local/bin
ENV LD_LIBRARY_PATH=/usr/local/lib
ARG LIBYAML_VERSION=0.2.5
RUN set -ex \
&& git clone https://github.com/yaml/libyaml.git \
&& cd libyaml \
&& git checkout $LIBYAML_VERSION \
&& ./bootstrap \
&& ./configure \
&& make \
&& make install \
&& cd .. \
&& rm -fr libyaml
WORKDIR /armada
COPY requirements.txt /tmp/
RUN \
pip3 install -r /tmp/requirements.txt && \
useradd -u 1000 -g users -d /armada armada && \
rm -rf /tmp/requirements.txt
COPY . /armada
RUN \
mv etc/armada /etc/ && \
cd /armada && \
chown -R armada:users /armada && \
python3 setup.py install
USER armada

4
images/armada/Dockerfile.ubuntu_xenial → images/armada/Dockerfile.ubuntu_focal
View File

@ -1,4 +1,4 @@
ARG FROM=ubuntu:16.04 ARG FROM=ubuntu:20.04
FROM ${FROM} FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
@ -70,7 +70,7 @@ RUN set -ex \
&& make install \ && make install \
&& cd .. \ && cd .. \
&& rm -fr libyaml \ && rm -fr libyaml \
&& python3 -m pip install -U 'pip<21.0' \ && python3 -m pip install -U pip \
&& pip3 install -r requirements.txt --no-cache-dir \ && pip3 install -r requirements.txt --no-cache-dir \
&& curl -fSSL -O ${HELM_ARTIFACT_URL} \ && curl -fSSL -O ${HELM_ARTIFACT_URL} \
&& tar -xvf $(basename ${HELM_ARTIFACT_URL}) \ && tar -xvf $(basename ${HELM_ARTIFACT_URL}) \

6
requirements.txt
View File

@ -1,11 +1,11 @@
amqp<2.7,>=2.6.0 amqp==5.0.8
deepdiff>=3.3.0 deepdiff>=3.3.0
gitpython gitpython
jsonschema>=3.0.1,<4 jsonschema>=3.0.1,<4
keystoneauth1>=3.18.0 keystoneauth1>=3.18.0
keystonemiddleware>=5.3.0 keystonemiddleware>=5.3.0
kombu<4.7,>=4.6.10 kombu==5.1.0
kubernetes~=25.3.0; python_version >= '3.6' kubernetes~=26.1.0; python_version >= '3.6'
Paste>=2.0.3 Paste>=2.0.3
PasteDeploy>=1.5.2 PasteDeploy>=1.5.2
pylibyaml~=0.1 pylibyaml~=0.1

11
setup.cfg
View File

@ -1,10 +1,11 @@
[metadata] [metadata]
name = Armada name = Armada
version = 1.0
summary = Tool for managing multiple Helm charts with dependencies by centralizing all configurations in a single Armada YAML and providing life-cycle hooks for all Helm releases. summary = Tool for managing multiple Helm charts with dependencies by centralizing all configurations in a single Armada YAML and providing life-cycle hooks for all Helm releases.
description-file = README.rst description_file = README.rst
author = The Airship Authors author = The Airship Authors
author-email = airship-discuss@lists.airshipit.org author_email = airship-discuss@lists.airshipit.org
home-page = https://airshipit.org home_page = https://airshipit.org
classifier = classifier =
Intended Audience :: Information Technology Intended Audience :: Information Technology
@ -13,8 +14,8 @@ classifier =
Operating System :: POSIX :: Linux Operating System :: POSIX :: Linux
Programming Language :: Python Programming Language :: Python
Programming Language :: Python :: 3 Programming Language :: Python :: 3
Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.10
[files] [files]
packages = packages =

11
tools/gate/playbooks/airskiff-deploy.yaml
View File

@ -13,6 +13,17 @@
# limitations under the License. # limitations under the License.
- hosts: primary - hosts: primary
vars:
env:
HTTP_PROXY: ""
HTTPS_PROXY: ""
NO_PROXY: ""
DISTRO: "{{ distro }}"
become: true
roles:
- clear-firewall
tasks: tasks:
# Stop systemd-resolved service before starting docker. # Stop systemd-resolved service before starting docker.
- name: stop systemd-resolved service - name: stop systemd-resolved service

4
tools/gate/playbooks/docker-image-build.yaml
View File

@ -72,7 +72,11 @@
name: "{{ item }}" name: "{{ item }}"
allow_unauthenticated: True allow_unauthenticated: True
with_items: with_items:
- apparmor
- docker-ce - docker-ce
- docker-ce-cli
- docker-buildx-plugin
- containerd.io
- python3-pip - python3-pip
- python3-setuptools - python3-setuptools
- pip: - pip:

2
tools/helm_tk.sh
View File

@ -17,7 +17,7 @@
set -eux set -eux
HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"} HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"f4972121bcb41c8d74748917804d2b239ab757f9"} HTK_STABLE_COMMIT=${HTK_COMMIT:-"a7cd689280cdbc0acd04a7a1b745941260e8700b"}
TMP_DIR=$(mktemp -d) TMP_DIR=$(mktemp -d)

10
tox.ini
View File

@ -7,7 +7,13 @@ envlist = py38, pep8, cover, bandit
deps= deps=
-r{toxinidir}/requirements.txt -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt -r{toxinidir}/test-requirements.txt
passenv=HTTP_PROXY,HTTPS_PROXY,http_proxy,https_proxy,NO_PROXY,no_proxy passenv=
HTTP_PROXY
HTTPS_PROXY
http_proxy
https_proxy
NO_PROXY
no_proxy
setenv= setenv=
VIRTUAL_ENV={envdir} VIRTUAL_ENV={envdir}
usedevelop = True usedevelop = True
@ -101,7 +107,7 @@ show-source = true
# [H904] Delay string interpolations at logging calls. # [H904] Delay string interpolations at logging calls.
enable-extensions = H106,H201,H904 enable-extensions = H106,H201,H904
# [W503] line break before binary operator # [W503] line break before binary operator
ignore = W503 ignore = W503,B202,B113
exclude = .git,.tox,dist,*lib/python*,*egg,build,releasenotes,doc/*,venv exclude = .git,.tox,dist,*lib/python*,*egg,build,releasenotes,doc/*,venv
max-complexity = 24 max-complexity = 24
application-import-names = armada application-import-names = armada