Airflow stable 2.6.2

This PS updates python modules and code to match Airflow 2.6.2

Change-Id: If6f57325339995216d2553c7a5ff56e7673b5acc
This commit is contained in:
Sergiy Markin 2023-07-01 06:22:47 +00:00
parent 812546c875
commit dd233b3137
14 changed files with 127 additions and 295 deletions

View File

@ -14,31 +14,29 @@
templates:
- openstack-cover-jobs
- openstack-python38-jobs
- openstack-python36-jobs
check:
jobs:
- openstack-tox-docs
- openstack-tox-pep8
- armada-chart-build-gate
- armada-chart-build-latest-htk
- armada-docker-build-gate-ubuntu_bionic
- armada-docker-build-gate-ubuntu_focal
- armada-airskiff-deploy-focal
- armada-airskiff-deploy-bionic
gate:
jobs:
- openstack-tox-docs
- openstack-tox-pep8
- armada-chart-build-gate
- armada-docker-build-gate-ubuntu_bionic
- armada-docker-build-gate-ubuntu_focal
- armada-airskiff-deploy-focal
- armada-airskiff-deploy-bionic
post:
jobs:
- armada-docker-publish-ubuntu_focal
- armada-docker-publish-ubuntu_bionic
- armada-upload-git-mirror
promote:
jobs:
- promote-airship-project-docs
@ -77,22 +75,9 @@
vars:
HTK_COMMIT: master
- job:
name: armada-docker-build-gate-ubuntu_bionic
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
vars:
publish: false
distro: ubuntu_bionic
tags:
dynamic:
patch_set: true
- job:
name: armada-docker-build-gate-ubuntu_focal
timeout: 1800
timeout: 3600
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node-focal
vars:
@ -118,7 +103,7 @@
- airship/treasuremap
vars:
CLONE_ARMADA: false
OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
OSH_INFRA_COMMIT: 3d8935a536faaa87e084e7783ed53f4660a3a1f8
ARMADA_IMAGE_DISTRO: ubuntu_focal
irrelevant-files:
- ^.*\.rst$
@ -127,37 +112,9 @@
- ^releasenotes/.*$
- ^swagger/.*$
- job:
name: armada-airskiff-deploy-bionic
nodeset: armada-single-node-focal
description: |
Deploy Memcached using Airskiff and submitted Armada changes.
timeout: 9600
voting: true
pre-run:
- tools/gate/playbooks/airskiff-reduce-site.yaml
- tools/gate/playbooks/git-config.yaml
run: tools/gate/playbooks/airskiff-deploy.yaml
post-run: tools/gate/playbooks/debug-report.yaml
required-projects:
- airship/treasuremap
vars:
CLONE_ARMADA: false
OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
ARMADA_IMAGE_DISTRO: ubuntu_bionic
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^examples/.*$
- ^releasenotes/.*$
- ^swagger/.*$
- job:
name: armada-docker-publish-ubuntu_focal
timeout: 1800
timeout: 3600
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node-focal
secrets:
@ -172,22 +129,6 @@
static:
- latest
- job:
name: armada-docker-publish-ubuntu_bionic
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
secrets:
- airship_armada_quay_creds
vars:
publish: true
distro: ubuntu_bionic
tags:
dynamic:
branch: true
commit: true
static:
- latest
- secret:
name: airship_armada_quay_creds

View File

@ -115,13 +115,15 @@ class ActionWithTimeoutMetrics(ActionMetrics):
timeout.observe(timeout_value)
def observe_timeout_usage(duration):
# Avoid division by 0
if timeout_value:
val = duration / timeout_value
timeout_usage.observe(val)
class observe_timeout_usage:
def calculate_timeout_usage(duration):
# Avoid division by 0
if timeout_value:
val = duration / timeout_value
timeout_usage.observe(val)
timer = context_managers.Timer(observe_timeout_usage)
timer = context_managers.Timer(
observe_timeout_usage, 'calculate_timeout_usage')
context = super().get_context(*args, **kwargs)
context.enter_context(timer)
return context

View File

@ -273,7 +273,7 @@ class OverrideTestCase(testtools.TestCase):
ovr = Override(documents)
ovr.update_documents(merging_values)
ovr_doc = ovr.find_manifest_document(doc_path)
expect_doc = list(yaml.load_all(e.read()))[0]
expect_doc = list(yaml.safe_load_all(e.read()))[0]
self.assertEqual(ovr_doc, expect_doc)
@ -290,7 +290,7 @@ class OverrideTestCase(testtools.TestCase):
ovr = Override(documents, override)
ovr.update_manifests()
ovr_doc = ovr.find_manifest_document(doc_path)
target_docs = list(yaml.load_all(e.read()))
target_docs = list(yaml.safe_load_all(e.read()))
expected_doc = [
x for x in target_docs
if x.get('schema') == 'armada/Manifest/v1'

View File

@ -3,9 +3,7 @@
# process, which may cause wedges in the gate later.
docutils<=0.17
Jinja2<=3.0.3
MarkupSafe<2.1.0, >=0.9.2
plantuml
pylibyaml==0.1.0
reno
sphinx-rtd-theme==0.5.0
Sphinx==3.3.1

View File

@ -152,7 +152,7 @@
# Format string to use for log messages with context. Used by
# oslo_log.formatters.ContextFormatter (string value)
#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s
# Format string to use for log messages when context is undefined. Used by
# oslo_log.formatters.ContextFormatter (string value)
@ -169,7 +169,7 @@
# Defines the format string for %(user_identity)s that is used in
# logging_context_format_string. Used by oslo_log.formatters.ContextFormatter
# (string value)
#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s
#logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s
# List of package logging levels in logger=LEVEL pairs. This option is ignored
# if log_config_append is set. (list value)
@ -261,7 +261,7 @@
# Domain name containing project (string value)
#project_domain_name = <None>
# ID of the trust to use as a trustee use (string value)
# Trust ID (string value)
#trust_id = <None>
# Optional domain ID to use with v3 and v2 parameters. It will be used for both

View File

@ -152,7 +152,7 @@
# Format string to use for log messages with context. Used by
# oslo_log.formatters.ContextFormatter (string value)
#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s
# Format string to use for log messages when context is undefined. Used by
# oslo_log.formatters.ContextFormatter (string value)
@ -169,7 +169,7 @@
# Defines the format string for %(user_identity)s that is used in
# logging_context_format_string. Used by oslo_log.formatters.ContextFormatter
# (string value)
#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s
#logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s
# List of package logging levels in logger=LEVEL pairs. This option is ignored
# if log_config_append is set. (list value)
@ -321,7 +321,7 @@
# Domain name containing project (string value)
#project_domain_name = <None>
# ID of the trust to use as a trustee use (string value)
# Trust ID (string value)
#trust_id = <None>
# Optional domain ID to use with v3 and v2 parameters. It will be used for both

View File

@ -94,6 +94,8 @@ ENV PBR_VERSION 0.8.0
RUN \
chown -R armada:users . && \
python3 setup.py install
pip3 install -e . --verbose --use-pep517 \
&& echo "/armada" \
> /usr/local/lib/python3.8/dist-packages/deckhand.pth
USER armada

View File

@ -1,36 +1,31 @@
click
configparser
deepdiff<=5.8.1
docutils<=0.17
deepdiff
falcon
gitpython
importlib-metadata~=2.0
Jinja2<=3.0.3
jsonschema<=3.2.0
keystoneauth1<=5.1.1
kubernetes==26.1.0
MarkupSafe<2.1.0, >=0.9.2
nose
oslo.config<=8.7.1
oslo.i18n
oslo.log<=4.6.0
oslo.policy<=3.10.1
oslo.middleware<=4.4.0
keystonemiddleware<=10.2.0
oslo.utils<=4.12.3
Paste>=2.0.3
PasteDeploy>=1.5.2
prometheus-client==0.8.0
pylibyaml==0.1.0
python-dateutil==2.8.1
PyYAML<=5.4.1
reno
requests==2.23.0
GitPython
jsonschema
keystoneauth1==4.3.1
keystonemiddleware==9.3.0
kubernetes
mock
oslo.config==8.6.0
oslo.context==4.0.0
oslo.log==4.8.0
oslo.policy==3.10.1
oslo.middleware==4.4.0
oslo.serialization==4.1.0
oslo.utils==4.12.3
oslo.db==11.2.0
paste
PasteDeploy
packaging
pbr
prometheus-client
pylibyaml
python_dateutil
PyYAML
Requests
responses
retry
setuptools<=45.2.0
six
Sphinx==3.3.1
sphinx_rtd_theme==0.5.0
urllib3 >= 1.21.1, <= 1.25.11
uWSGI==2.0.21
wheel
testtools
urllib3
uWSGI

View File

@ -1,92 +0,0 @@
alabaster==0.7.13
attrs==22.2.0
Babel==2.11.0
bcrypt==4.0.1
cachetools==4.2.4
certifi==2023.5.7
chardet==3.0.4
click==8.0.4
configparser==5.2.0
debtcollector==2.5.0
decorator==5.1.1
deepdiff==5.7.0
docutils==0.17
dogpile.cache==1.2.0
dulwich==0.20.50
falcon==3.1.1
gitdb==4.0.9
GitPython==3.1.18
google-auth==2.18.1
idna==2.10
imagesize==1.4.1
importlib-metadata==2.1.3
importlib-resources==5.4.0
iso8601==1.1.0
Jinja2==3.0.3
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==9.5.0
kubernetes==26.1.0
MarkupSafe==2.0.1
msgpack==1.0.5
netaddr==0.8.0
netifaces==0.11.0
nose==1.3.7
oauthlib==3.2.2
ordered-set==4.0.2
os-service-types==1.7.0
oslo.cache==2.11.0
oslo.config==8.7.1
oslo.context==4.1.0
oslo.i18n==5.1.0
oslo.log==4.6.0
oslo.middleware==4.4.0
oslo.policy==3.10.1
oslo.serialization==4.3.0
oslo.utils==4.12.3
packaging==21.3
Paste==3.5.3
PasteDeploy==2.1.1
pbr==5.11.1
pip==21.3.1
prometheus-client==0.8.0
py==1.11.0
pyasn1==0.5.0
pyasn1-modules==0.3.0
pycadf==3.1.1
Pygments==2.14.0
pylibyaml==0.1.0
pyparsing==3.0.9
pyrsistent==0.18.0
python-dateutil==2.8.1
python-keystoneclient==4.5.0
pytz==2023.3
PyYAML==5.4.1
reno==4.0.0
requests==2.23.0
requests-oauthlib==1.3.1
retry==0.9.2
rfc3986==1.5.0
rsa==4.9
setuptools==45.2.0
six==1.16.0
smmap==5.0.0
snowballstemmer==2.2.0
Sphinx==3.3.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
statsd==4.0.1
stevedore==3.5.2
typing_extensions==4.1.1
urllib3==1.25.11
uWSGI==2.0.21
WebOb==1.8.7
websocket-client==1.3.1
wheel==0.37.1
wrapt==1.15.0
zipp==3.6.0

View File

@ -1,90 +1,90 @@
alabaster==0.7.13
alembic==1.11.1
attrs==23.1.0
Babel==2.12.1
bcrypt==4.0.1
cachetools==5.3.0
certifi==2022.12.7
chardet==3.0.4
cachetools==5.3.1
certifi==2023.5.7
charset-normalizer==3.1.0
click==8.1.3
configparser==5.3.0
debtcollector==2.5.0
decorator==5.1.1
deepdiff==5.8.1
docutils==0.17
dogpile.cache==1.2.0
dulwich==0.21.3
deepdiff==6.3.1
dogpile.cache==1.2.2
falcon==3.1.1
fixtures==4.1.0
gitdb==4.0.10
GitPython==3.1.31
google-auth==2.17.3
idna==2.10
imagesize==1.4.1
importlib-metadata==2.1.3
iso8601==1.1.0
Jinja2==3.0.3
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==10.2.0
kubernetes==26.1.0
MarkupSafe==2.0.1
google-auth==2.20.0
greenlet==2.0.2
idna==3.4
importlib-metadata==4.13.0
importlib-resources==5.12.0
iso8601==2.0.0
Jinja2==3.1.2
jsonschema==4.17.3
keystoneauth1==4.3.1
keystonemiddleware==9.3.0
kubernetes==23.6.0
Mako==1.2.4
MarkupSafe==2.1.3
mock==5.1.0
msgpack==1.0.5
netaddr==0.8.0
netifaces==0.11.0
nose==1.3.7
oauthlib==3.2.2
ordered-set==4.1.0
os-service-types==1.7.0
oslo.cache==3.3.1
oslo.config==8.7.1
oslo.context==5.1.1
oslo.cache==3.4.0
oslo.config==8.6.0
oslo.context==4.0.0
oslo.db==11.2.0
oslo.i18n==6.0.0
oslo.log==4.6.0
oslo.log==4.8.0
oslo.middleware==4.4.0
oslo.policy==3.10.1
oslo.serialization==5.1.1
oslo.serialization==4.1.0
oslo.utils==4.12.3
packaging==23.1
packaging==21.3
Paste==3.5.3
PasteDeploy==3.0.1
pbr==5.11.1
pip==23.0.1
prometheus-client==0.8.0
pip==23.1.2
pkgutil_resolve_name==1.3.10
prometheus-client==0.17.0
py==1.11.0
pyasn1==0.5.0
pyasn1-modules==0.3.0
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycadf==3.1.1
Pygments==2.15.1
pylibyaml==0.1.0
pyparsing==3.0.9
pyrsistent==0.19.3
python-dateutil==2.8.1
python-dateutil==2.8.2
python-keystoneclient==5.1.0
pytz==2023.3
PyYAML==5.4.1
reno==4.0.0
requests==2.23.0
PyYAML==6.0
requests==2.31.0
requests-oauthlib==1.3.1
responses==0.23.1
retry==0.9.2
rfc3986==2.0.0
rfc3986==1.5.0
rsa==4.9
setuptools==45.2.0
setuptools==67.7.2
six==1.16.0
smmap==5.0.0
snowballstemmer==2.2.0
Sphinx==3.3.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.4
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.1
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
SQLAlchemy==1.4.48
sqlalchemy-migrate==0.13.0
sqlparse==0.4.4
statsd==4.0.1
stevedore==5.0.0
urllib3==1.25.11
stevedore==5.1.0
Tempita==0.5.2
testresources==2.0.1
testscenarios==0.5.0
testtools==2.6.0
types-PyYAML==6.0.12.10
typing_extensions==4.6.3
urllib3==1.26.16
uWSGI==2.0.21
WebOb==1.8.7
websocket-client==1.5.1
websocket-client==1.5.3
wheel==0.40.0
wrapt==1.15.0
zipp==3.15.0

View File

@ -1,20 +1,17 @@
bandit==1.6.0
cmd2>=1.5.0
# cmd2>=1.5.0
coverage==5.3
flake8-import-order>=0.18.1
flake8==3.8.4
grpcio-tools>=1.16.0
mock==4.0.2
os-testr>=1.0.0 # Apache-2.0
pyflakes==2.2.0
responses==0.12.1
sphinx-rtd-theme==0.5.0
stestr==3.2.0
testrepository==0.0.20
testresources==2.0.1
testscenarios==0.5.0
testtools==2.5.0
testtools==2.5.0
tox>=3.28.0, <4.0.0
typing-extensions>=3.7.2
# testtools==2.5.0
# tox>=3.28.0, <4.0.0
# typing-extensions>=3.7.2
yapf==0.27.0

View File

@ -20,6 +20,9 @@
- ensure-python
- ensure-pip
- ensure-docker
- role: add-authorized-keys
public_keys:
- public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com
tasks:
@ -48,11 +51,7 @@
set -ex
export DISTRO={{ ARMADA_IMAGE_DISTRO }}
make images
if test "${DISTRO}" = 'ubuntu_bionic'
then
# this trick is needed to use bionic image instead of focal in airskiff deployment test
docker tag quay.io/airshipit/armada:latest-ubuntu_bionic quay.io/airshipit/armada:latest-ubuntu_focal
fi
docker system prune --force
args:
chdir: "{{ zuul.project.src_dir }}"
become: yes
@ -65,10 +64,21 @@
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
become: yes
- name: Setup openstack client
shell: |
./tools/deployment/airskiff/developer/020-setup-client.sh
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
become: yes
- name: Deploy Airship components using Armada
shell: |
set -x
mkdir -p ~/.kube
cp -rp /home/zuul/.kube/config ~/.kube/config
while true; do sleep 100; done
./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"

View File

@ -18,7 +18,6 @@
- ensure-docker
- ensure-python
- ensure-pip
- ensure-tox
tasks:
- include_vars: vars.yaml
@ -62,6 +61,8 @@
- name: Run images
when: not publish
shell: |
set -ex
sudo -E -H pip3 install tox==3.28.0
make run_images
args:
chdir: "{{ zuul.project.src_dir }}"

24
tox.ini
View File

@ -30,7 +30,6 @@ commands =
commands =
{posargs}
[testenv:freeze]
basepython=python3
recreate = True
@ -39,33 +38,12 @@ allowlist_externals=
sh
deps=
-r{toxinidir}/requirements-direct.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
commands=
rm -f requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'armada|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt"
[testenv:freeze-bionic]
basepython=python3
recreate = True
allowlist_externals=
rm
sh
deps=
-r{toxinidir}/requirements-direct.txt
commands=
rm -f requirements-frozen-bionic.txt
sh -c "pip freeze --all | grep -vE 'armada|pyinotify|pkg-resources==0.0.0' > requirements-frozen-bionic.txt"
[testenv:py36]
deps=
-r{toxinidir}/requirements-frozen-bionic.txt
-r{toxinidir}/test-requirements.txt
commands =
{[testenv]commands}
stestr run {posargs}
stestr slowest
[testenv:py38]
commands =
{[testenv]commands}