airship
/
armada
Code Issues Proposed changes

Airflow stable 2.6.2 

This PS updates python modules and code to match Airflow 2.6.2

Change-Id: If6f57325339995216d2553c7a5ff56e7673b5acc
This commit is contained in:
Sergiy Markin 2023-07-01 06:22:47 +00:00
parent 812546c875
commit 33a3e46830
12 changed files with 105 additions and 297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
.zuul.yaml
View File

@ -14,31 +14,29 @@
templates:
- openstack-cover-jobs
- openstack-python38-jobs
- openstack-python36-jobs
check:
jobs:
- openstack-tox-docs
- openstack-tox-pep8
- armada-chart-build-gate
- armada-chart-build-latest-htk
- armada-docker-build-gate-ubuntu_bionic
- armada-docker-build-gate-ubuntu_focal
- armada-airskiff-deploy-focal
- armada-airskiff-deploy-bionic
gate:
jobs:
- openstack-tox-docs
- openstack-tox-pep8
- armada-chart-build-gate
- armada-docker-build-gate-ubuntu_bionic
- armada-docker-build-gate-ubuntu_focal
- armada-airskiff-deploy-focal
- armada-airskiff-deploy-bionic
post:
jobs:
- armada-docker-publish-ubuntu_focal
- armada-docker-publish-ubuntu_bionic
- armada-upload-git-mirror
promote:
jobs:
- promote-airship-project-docs
@ -77,19 +75,6 @@
vars:
HTK_COMMIT: master
- job:
name: armada-docker-build-gate-ubuntu_bionic
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
vars:
publish: false
distro: ubuntu_bionic
tags:
dynamic:
patch_set: true
- job:
name: armada-docker-build-gate-ubuntu_focal
timeout: 1800
@ -118,7 +103,7 @@
- airship/treasuremap
vars:
CLONE_ARMADA: false
OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
OSH_INFRA_COMMIT: 3d8935a536faaa87e084e7783ed53f4660a3a1f8
ARMADA_IMAGE_DISTRO: ubuntu_focal
irrelevant-files:
- ^.*\.rst$
@ -127,34 +112,6 @@
- ^releasenotes/.*$
- ^swagger/.*$
- job:
name: armada-airskiff-deploy-bionic
nodeset: armada-single-node-focal
description: |
Deploy Memcached using Airskiff and submitted Armada changes.
timeout: 9600
voting: true
pre-run:
- tools/gate/playbooks/airskiff-reduce-site.yaml
- tools/gate/playbooks/git-config.yaml
run: tools/gate/playbooks/airskiff-deploy.yaml
post-run: tools/gate/playbooks/debug-report.yaml
required-projects:
- airship/treasuremap
vars:
CLONE_ARMADA: false
OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
ARMADA_IMAGE_DISTRO: ubuntu_bionic
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^examples/.*$
- ^releasenotes/.*$
- ^swagger/.*$
- job:
name: armada-docker-publish-ubuntu_focal
timeout: 1800
@ -172,22 +129,6 @@
static:
- latest
- job:
name: armada-docker-publish-ubuntu_bionic
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
secrets:
- airship_armada_quay_creds
vars:
publish: true
distro: ubuntu_bionic
tags:
dynamic:
branch: true
commit: true
static:
- latest
- secret:
name: airship_armada_quay_creds

14
armada/handlers/metrics.py
View File

@ -115,13 +115,15 @@ class ActionWithTimeoutMetrics(ActionMetrics):
timeout.observe(timeout_value)
def observe_timeout_usage(duration):
# Avoid division by 0
if timeout_value:
val = duration / timeout_value
timeout_usage.observe(val)
class observe_timeout_usage:
def calculate_timeout_usage(duration):
# Avoid division by 0
if timeout_value:
val = duration / timeout_value
timeout_usage.observe(val)
timer = context_managers.Timer(observe_timeout_usage)
timer = context_managers.Timer(
observe_timeout_usage, 'calculate_timeout_usage')
context = super().get_context(*args, **kwargs)
context.enter_context(timer)
return context

4
armada/tests/unit/handlers/test_override.py
View File

@ -273,7 +273,7 @@ class OverrideTestCase(testtools.TestCase):
ovr = Override(documents)
ovr.update_documents(merging_values)
ovr_doc = ovr.find_manifest_document(doc_path)
expect_doc = list(yaml.load_all(e.read()))[0]
expect_doc = list(yaml.safe_load_all(e.read()))[0]
self.assertEqual(ovr_doc, expect_doc)
@ -290,7 +290,7 @@ class OverrideTestCase(testtools.TestCase):
ovr = Override(documents, override)
ovr.update_manifests()
ovr_doc = ovr.find_manifest_document(doc_path)
target_docs = list(yaml.load_all(e.read()))
target_docs = list(yaml.safe_load_all(e.read()))
expected_doc = [
x for x in target_docs
if x.get('schema') == 'armada/Manifest/v1'

2
doc/requirements.txt
View File

@ -3,9 +3,7 @@
# process, which may cause wedges in the gate later.
docutils<=0.17
Jinja2<=3.0.3
MarkupSafe<2.1.0, >=0.9.2
plantuml
pylibyaml==0.1.0
reno
sphinx-rtd-theme==0.5.0
Sphinx==3.3.1

6
doc/source/_static/armada.conf.sample
View File

@ -152,7 +152,7 @@
# Format string to use for log messages with context. Used by
# oslo_log.formatters.ContextFormatter (string value)
#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s
# Format string to use for log messages when context is undefined. Used by
# oslo_log.formatters.ContextFormatter (string value)
@ -169,7 +169,7 @@
# Defines the format string for %(user_identity)s that is used in
# logging_context_format_string. Used by oslo_log.formatters.ContextFormatter
# (string value)
#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s
#logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s
# List of package logging levels in logger=LEVEL pairs. This option is ignored
# if log_config_append is set. (list value)
@ -261,7 +261,7 @@
# Domain name containing project (string value)
#project_domain_name = <None>
# ID of the trust to use as a trustee use (string value)
# Trust ID (string value)
#trust_id = <None>
# Optional domain ID to use with v3 and v2 parameters. It will be used for both

4
images/armada/Dockerfile.ubuntu_focal
View File

@ -94,6 +94,8 @@ ENV PBR_VERSION 0.8.0
RUN \
chown -R armada:users . && \
python3 setup.py install
pip3 install -e . --verbose --use-pep517 \
&& echo "/armada" \
> /usr/local/lib/python3.8/dist-packages/deckhand.pth
USER armada

57
requirements-direct.txt
View File

@ -1,36 +1,27 @@
click
configparser
deepdiff<=5.8.1
docutils<=0.17
deepdiff
falcon
gitpython
importlib-metadata~=2.0
Jinja2<=3.0.3
jsonschema<=3.2.0
keystoneauth1<=5.1.1
kubernetes==26.1.0
MarkupSafe<2.1.0, >=0.9.2
nose
oslo.config<=8.7.1
oslo.i18n
oslo.log<=4.6.0
oslo.policy<=3.10.1
oslo.middleware<=4.4.0
keystonemiddleware<=10.2.0
oslo.utils<=4.12.3
Paste>=2.0.3
PasteDeploy>=1.5.2
prometheus-client==0.8.0
pylibyaml==0.1.0
python-dateutil==2.8.1
PyYAML<=5.4.1
reno
requests==2.23.0
GitPython
jsonschema
keystoneauth1==4.3.1
kubernetes
mock
oslo.config==8.6.0
oslo.context==4.0.0
oslo.log==4.8.0
oslo.policy==3.10.1
oslo.serialization==4.1.0
oslo.utils==4.12.3
oslo.db==11.2.0
packaging
pbr
prometheus-client
pylibyaml
python_dateutil
PyYAML
Requests
responses
retry
setuptools<=45.2.0
six
Sphinx==3.3.1
sphinx_rtd_theme==0.5.0
urllib3 >= 1.21.1, <= 1.25.11
uWSGI==2.0.21
wheel
testtools
urllib3
uWSGI

92
requirements-frozen-bionic.txt
View File

@ -1,92 +0,0 @@
alabaster==0.7.13
attrs==22.2.0
Babel==2.11.0
bcrypt==4.0.1
cachetools==4.2.4
certifi==2023.5.7
chardet==3.0.4
click==8.0.4
configparser==5.2.0
debtcollector==2.5.0
decorator==5.1.1
deepdiff==5.7.0
docutils==0.17
dogpile.cache==1.2.0
dulwich==0.20.50
falcon==3.1.1
gitdb==4.0.9
GitPython==3.1.18
google-auth==2.18.1
idna==2.10
imagesize==1.4.1
importlib-metadata==2.1.3
importlib-resources==5.4.0
iso8601==1.1.0
Jinja2==3.0.3
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==9.5.0
kubernetes==26.1.0
MarkupSafe==2.0.1
msgpack==1.0.5
netaddr==0.8.0
netifaces==0.11.0
nose==1.3.7
oauthlib==3.2.2
ordered-set==4.0.2
os-service-types==1.7.0
oslo.cache==2.11.0
oslo.config==8.7.1
oslo.context==4.1.0
oslo.i18n==5.1.0
oslo.log==4.6.0
oslo.middleware==4.4.0
oslo.policy==3.10.1
oslo.serialization==4.3.0
oslo.utils==4.12.3
packaging==21.3
Paste==3.5.3
PasteDeploy==2.1.1
pbr==5.11.1
pip==21.3.1
prometheus-client==0.8.0
py==1.11.0
pyasn1==0.5.0
pyasn1-modules==0.3.0
pycadf==3.1.1
Pygments==2.14.0
pylibyaml==0.1.0
pyparsing==3.0.9
pyrsistent==0.18.0
python-dateutil==2.8.1
python-keystoneclient==4.5.0
pytz==2023.3
PyYAML==5.4.1
reno==4.0.0
requests==2.23.0
requests-oauthlib==1.3.1
retry==0.9.2
rfc3986==1.5.0
rsa==4.9
setuptools==45.2.0
six==1.16.0
smmap==5.0.0
snowballstemmer==2.2.0
Sphinx==3.3.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
statsd==4.0.1
stevedore==3.5.2
typing_extensions==4.1.1
urllib3==1.25.11
uWSGI==2.0.21
WebOb==1.8.7
websocket-client==1.3.1
wheel==0.37.1
wrapt==1.15.0
zipp==3.6.0

106
requirements-frozen.txt
View File

@ -1,90 +1,78 @@
alabaster==0.7.13
alembic==1.11.1
attrs==23.1.0
Babel==2.12.1
bcrypt==4.0.1
cachetools==5.3.0
certifi==2022.12.7
chardet==3.0.4
cachetools==5.3.1
certifi==2023.5.7
charset-normalizer==3.1.0
click==8.1.3
configparser==5.3.0
debtcollector==2.5.0
decorator==5.1.1
deepdiff==5.8.1
docutils==0.17
dogpile.cache==1.2.0
dulwich==0.21.3
deepdiff==6.3.1
falcon==3.1.1
fixtures==4.1.0
gitdb==4.0.10
GitPython==3.1.31
google-auth==2.17.3
idna==2.10
imagesize==1.4.1
importlib-metadata==2.1.3
iso8601==1.1.0
Jinja2==3.0.3
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==10.2.0
kubernetes==26.1.0
MarkupSafe==2.0.1
google-auth==2.20.0
greenlet==2.0.2
idna==3.4
importlib-metadata==4.13.0
importlib-resources==5.12.0
iso8601==2.0.0
jsonschema==4.17.3
keystoneauth1==4.3.1
kubernetes==23.6.0
Mako==1.2.4
MarkupSafe==2.1.3
mock==5.1.0
msgpack==1.0.5
netaddr==0.8.0
netifaces==0.11.0
nose==1.3.7
oauthlib==3.2.2
ordered-set==4.1.0
os-service-types==1.7.0
oslo.cache==3.3.1
oslo.config==8.7.1
oslo.context==5.1.1
oslo.config==8.6.0
oslo.context==4.0.0
oslo.db==11.2.0
oslo.i18n==6.0.0
oslo.log==4.6.0
oslo.middleware==4.4.0
oslo.log==4.8.0
oslo.policy==3.10.1
oslo.serialization==5.1.1
oslo.serialization==4.1.0
oslo.utils==4.12.3
packaging==23.1
Paste==3.5.3
PasteDeploy==3.0.1
packaging==21.3
pbr==5.11.1
pip==23.0.1
prometheus-client==0.8.0
pip==23.1.2
pkgutil_resolve_name==1.3.10
prometheus-client==0.17.0
py==1.11.0
pyasn1==0.5.0
pyasn1-modules==0.3.0
pycadf==3.1.1
Pygments==2.15.1
pyasn1==0.4.8
pyasn1-modules==0.2.8
pylibyaml==0.1.0
pyparsing==3.0.9
pyrsistent==0.19.3
python-dateutil==2.8.1
python-keystoneclient==5.1.0
python-dateutil==2.8.2
pytz==2023.3
PyYAML==5.4.1
reno==4.0.0
requests==2.23.0
PyYAML==6.0
requests==2.31.0
requests-oauthlib==1.3.1
responses==0.23.1
retry==0.9.2
rfc3986==2.0.0
rfc3986==1.5.0
rsa==4.9
setuptools==45.2.0
setuptools==67.7.2
six==1.16.0
smmap==5.0.0
snowballstemmer==2.2.0
Sphinx==3.3.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.4
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.1
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
statsd==4.0.1
stevedore==5.0.0
urllib3==1.25.11
SQLAlchemy==1.4.48
sqlalchemy-migrate==0.13.0
sqlparse==0.4.4
stevedore==5.1.0
Tempita==0.5.2
testresources==2.0.1
testscenarios==0.5.0
testtools==2.6.0
types-PyYAML==6.0.12.10
typing_extensions==4.6.3
urllib3==1.26.16
uWSGI==2.0.21
WebOb==1.8.7
websocket-client==1.5.1
websocket-client==1.5.3
wheel==0.40.0
wrapt==1.15.0
zipp==3.15.0

11
test-requirements.txt
View File

@ -1,20 +1,17 @@
bandit==1.6.0
cmd2>=1.5.0
# cmd2>=1.5.0
coverage==5.3
flake8-import-order>=0.18.1
flake8==3.8.4
grpcio-tools>=1.16.0
mock==4.0.2
os-testr>=1.0.0 # Apache-2.0
pyflakes==2.2.0
responses==0.12.1
sphinx-rtd-theme==0.5.0
stestr==3.2.0
testrepository==0.0.20
testresources==2.0.1
testscenarios==0.5.0
testtools==2.5.0
testtools==2.5.0
tox>=3.28.0, <4.0.0
typing-extensions>=3.7.2
# testtools==2.5.0
# tox>=3.28.0, <4.0.0
# typing-extensions>=3.7.2
yapf==0.27.0

13
tools/gate/playbooks/airskiff-deploy.yaml
View File

@ -48,11 +48,7 @@
set -ex
export DISTRO={{ ARMADA_IMAGE_DISTRO }}
make images
if test "${DISTRO}" = 'ubuntu_bionic'
then
# this trick is needed to use bionic image instead of focal in airskiff deployment test
docker tag quay.io/airshipit/armada:latest-ubuntu_bionic quay.io/airshipit/armada:latest-ubuntu_focal
fi
docker system prune --force
args:
chdir: "{{ zuul.project.src_dir }}"
become: yes
@ -65,6 +61,13 @@
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
become: yes
- name: Setup openstack client
shell: |
./tools/deployment/airskiff/developer/020-setup-client.sh
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
become: yes
- name: Deploy Airship components using Armada
shell: |
mkdir -p ~/.kube

24
tox.ini
View File

@ -30,7 +30,6 @@ commands =
commands =
{posargs}
[testenv:freeze]
basepython=python3
recreate = True
@ -39,33 +38,12 @@ allowlist_externals=
sh
deps=
-r{toxinidir}/requirements-direct.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
commands=
rm -f requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'armada|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt"
[testenv:freeze-bionic]
basepython=python3
recreate = True
allowlist_externals=
rm
sh
deps=
-r{toxinidir}/requirements-direct.txt
commands=
rm -f requirements-frozen-bionic.txt
sh -c "pip freeze --all | grep -vE 'armada|pyinotify|pkg-resources==0.0.0' > requirements-frozen-bionic.txt"
[testenv:py36]
deps=
-r{toxinidir}/requirements-frozen-bionic.txt
-r{toxinidir}/test-requirements.txt
commands =
{[testenv]commands}
stestr run {posargs}
stestr slowest
[testenv:py38]
commands =
{[testenv]commands}