From f99abfa4337f8cbb591513aac404b11208d4187c Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Mon, 3 Jul 2023 20:35:08 +0000 Subject: [PATCH] Airflow stable 2.6.2 This PS updates python modules and code to match Airflow 2.6.2: - bionic py36 gates were removed - python code corrected to match new modules versions - selection of python modules versions was perfrmed based on airflow-2.6.2 constraints - postgresql image updated to 14.8 Change-Id: Ibdcc75e600166c20b842508aa5539587cca466f0 --- .zuul.yaml | 43 +----- charts/drydock/values.yaml | 2 +- doc/source/_static/drydock.conf.sample | 2 +- etc/drydock/drydock.conf.sample | 2 +- .../drivers/node/maasdriver/actions/node.py | 8 +- python/requirements-direct.txt | 16 ++- python/requirements-frozen-bionic.txt | 99 -------------- python/requirements-frozen.txt | 127 ++++++++---------- python/test-requirements.txt | 10 +- python/tests/postgres/start_postgres.sh | 6 +- tools/drydock_image_run.sh | 6 +- tools/helm_install.sh | 2 +- tox.ini | 36 +---- 13 files changed, 89 insertions(+), 270 deletions(-) delete mode 100644 python/requirements-frozen-bionic.txt diff --git a/.zuul.yaml b/.zuul.yaml index f88f18bc..70b5049b 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -14,7 +14,6 @@ templates: - docs-on-readthedocs - openstack-python38-jobs - - openstack-python36-jobs vars: rtd_webhook_id: '38809' rtd_project_name: 'airship-drydock' @@ -26,7 +25,6 @@ - airship-drydock-chart-build-gate - airship-drydock-chart-build-latest-htk - airship-drydock-docker-build-gate-ubuntu_focal - - airship-drydock-docker-build-gate-ubuntu_bionic gate: jobs: @@ -35,11 +33,10 @@ - airship-drydock-baclient-test - airship-drydock-chart-build-gate - airship-drydock-docker-build-gate-ubuntu_focal - - airship-drydock-docker-build-gate-ubuntu_bionic + post: jobs: - airship-drydock-docker-publish-ubuntu_focal - - airship-drydock-docker-publish-ubuntu_bionic - drydock-upload-git-mirror - nodeset: @@ -84,7 +81,7 @@ run: tools/gate/playbooks/build-charts.yaml nodeset: airship-drydock-single-node-focal vars: - HTK_COMMIT: a7cd689280cdbc0acd04a7a1b745941260e8700b + HTK_COMMIT: 97ce6d7d8e9a090c748800d69a57bbd9af698b60 - job: name: airship-drydock-chart-build-latest-htk @@ -113,21 +110,6 @@ patch_set: true -- job: - name: airship-drydock-docker-build-gate-ubuntu_bionic - timeout: 3600 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: airship-drydock-single-node - irrelevant-files: - - '^doc/.*' - - '^charts/.*' - vars: - publish: false - distro: ubuntu_bionic - tags: - dynamic: - patch_set: true - - job: name: airship-drydock-docker-publish-ubuntu_focal @@ -148,26 +130,7 @@ commit: true static: - latest - -- job: - name: airship-drydock-docker-publish-ubuntu_bionic - timeout: 3600 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: airship-drydock-single-node-focal - secrets: - - airship_drydock_quay_creds - irrelevant-files: - - '^doc/.*' - - '^charts/.*' - vars: - publish: true - distro: ubuntu_bionic - tags: - dynamic: - branch: true - commit: true - static: - - latest + - airflow_2.6.2 - secret: diff --git a/charts/drydock/values.yaml b/charts/drydock/values.yaml index b5e65935..59e405f6 100644 --- a/charts/drydock/values.yaml +++ b/charts/drydock/values.yaml @@ -35,7 +35,7 @@ images: ks_user: docker.io/openstackhelm/heat:newton ks_service: docker.io/openstackhelm/heat:newton ks_endpoints: docker.io/openstackhelm/heat:newton - drydock_db_init: docker.io/postgres:14.6 + drydock_db_init: docker.io/postgres:14.8 drydock_db_cleanup: quay.io/airshipit/drydock:master drydock_db_sync: quay.io/airshipit/drydock:master pull_policy: "IfNotPresent" diff --git a/doc/source/_static/drydock.conf.sample b/doc/source/_static/drydock.conf.sample index 323d4df9..b04f5056 100644 --- a/doc/source/_static/drydock.conf.sample +++ b/doc/source/_static/drydock.conf.sample @@ -78,7 +78,7 @@ # Domain name containing project (string value) #project_domain_name = -# ID of the trust to use as a trustee use (string value) +# Trust ID (string value) #trust_id = # Optional domain ID to use with v3 and v2 parameters. It will be used for both diff --git a/etc/drydock/drydock.conf.sample b/etc/drydock/drydock.conf.sample index 323d4df9..b04f5056 100644 --- a/etc/drydock/drydock.conf.sample +++ b/etc/drydock/drydock.conf.sample @@ -78,7 +78,7 @@ # Domain name containing project (string value) #project_domain_name = -# ID of the trust to use as a trustee use (string value) +# Trust ID (string value) #trust_id = # Optional domain ID to use with v3 and v2 parameters. It will be used for both diff --git a/python/drydock_provisioner/drivers/node/maasdriver/actions/node.py b/python/drydock_provisioner/drivers/node/maasdriver/actions/node.py index 0d1830c4..71b42c14 100644 --- a/python/drydock_provisioner/drivers/node/maasdriver/actions/node.py +++ b/python/drydock_provisioner/drivers/node/maasdriver/actions/node.py @@ -296,7 +296,7 @@ class DestroyNode(BaseMaasAction): ctx_type='node') self.task.success(focus=n.get_id()) continue - elif type(machine) == maas_rack.RackController: + elif type(machine) is maas_rack.RackController: msg = "Cannot delete rack controller {}.".format(n.name) self.logger.info(msg) self.task.add_status_msg(msg=msg, @@ -1130,7 +1130,7 @@ class IdentifyNode(BaseMaasAction): error=True, ctx=n.name, ctx_type='node') - elif type(machine) == maas_machine.Machine: + elif type(machine) is maas_machine.Machine: machine.update_identity( n, domain=n.get_domain(site_design), @@ -1144,7 +1144,7 @@ class IdentifyNode(BaseMaasAction): ctx=n.name, ctx_type='node') self.task.success(focus=n.get_id()) - elif type(machine) == maas_rack.RackController: + elif type(machine) is maas_rack.RackController: msg = "Rack controller %s identified in MaaS" % n.name self.logger.debug(msg) self.task.add_status_msg(msg=msg, @@ -1219,7 +1219,7 @@ class ConfigureHardware(BaseMaasAction): self.logger.debug("Locating node %s for commissioning" % (n.name)) machine = find_node_in_maas(self.maas_client, n) - if type(machine) == maas_rack.RackController: + if type(machine) is maas_rack.RackController: msg = "Located node %s in MaaS as rack controller. Skipping." % ( n.name) self.logger.info(msg) diff --git a/python/requirements-direct.txt b/python/requirements-direct.txt index 3799f392..1cc7be0d 100644 --- a/python/requirements-direct.txt +++ b/python/requirements-direct.txt @@ -7,13 +7,19 @@ falcon iso8601 jinja2 jsonschema -keystoneauth1 +keystoneauth1==4.3.1 keystonemiddleware libvirt-python netaddr +mock oauthlib -oslo.config -oslo.policy +oslo.config==8.6.0 +oslo.context==4.0.0 +oslo.log==4.8.0 +oslo.policy==3.10.1 +oslo.serialization==4.1.0 +oslo.utils==4.12.3 +oslo.db==11.2.0 oslo.versionedobjects Paste PasteDeploy @@ -22,9 +28,11 @@ PTable pyghmi pylibyaml pymongo +python-memcached PyYAML redfish requests -SQLAlchemy<=1.3.20 +responses +SQLAlchemy ulid2 uWSGI \ No newline at end of file diff --git a/python/requirements-frozen-bionic.txt b/python/requirements-frozen-bionic.txt deleted file mode 100644 index 382b98cc..00000000 --- a/python/requirements-frozen-bionic.txt +++ /dev/null @@ -1,99 +0,0 @@ -alembic==1.7.7 -amqp==5.1.1 -attrs==22.2.0 -bcrypt==4.0.1 -Beaker==1.12.1 -cached-property==1.5.2 -cachetools==4.2.4 -certifi==2023.5.7 -cffi==1.15.1 -charset-normalizer==2.0.12 -click==8.0.4 -cryptography==40.0.2 -debtcollector==2.5.0 -decorator==5.1.1 -defusedxml==0.7.1 -dnspython==2.2.1 -dogpile.cache==1.2.1 -eventlet==0.33.3 -falcon==3.1.1 -fasteners==0.18 -fixtures==4.0.1 -futurist==2.4.1 -greenlet==2.0.2 -idna==3.4 -importlib-metadata==4.8.3 -importlib-resources==5.4.0 -iso8601==1.1.0 -Jinja2==3.0.3 -jsonpatch==1.32 -jsonpath-rw==1.4.0 -jsonpointer==2.3 -jsonschema==3.2.0 -keystoneauth1==5.1.2 -keystonemiddleware==9.5.0 -kombu==5.1.0 -libvirt-python==9.3.0 -Mako==1.1.6 -MarkupSafe==2.0.1 -msgpack==1.0.5 -netaddr==0.8.0 -netifaces==0.11.0 -oauthlib==3.2.2 -os-service-types==1.7.0 -oslo.cache==2.11.0 -oslo.concurrency==4.5.1 -oslo.config==8.8.0 -oslo.context==4.1.0 -oslo.i18n==5.1.0 -oslo.log==4.8.0 -oslo.messaging==12.14.0 -oslo.metrics==0.4.0 -oslo.middleware==4.5.1 -oslo.policy==3.12.1 -oslo.serialization==4.3.0 -oslo.service==2.8.0 -oslo.utils==4.13.0 -oslo.versionedobjects==2.6.0 -packaging==21.3 -Paste==3.5.3 -PasteDeploy==2.1.1 -pbr==5.11.1 -pip==21.3.1 -ply==3.11 -prometheus-client==0.16.0 -psycopg2-binary==2.9.6 -PTable==0.9.2 -pycadf==3.1.1 -pycparser==2.21 -pyghmi==1.5.61 -pylibyaml==0.1.0 -pymongo==4.1.1 -pyparsing==3.0.9 -pyrsistent==0.18.0 -python-dateutil==2.8.2 -python-keystoneclient==4.5.0 -pytz==2023.3 -PyYAML==6.0 -redfish==3.1.9 -repoze.lru==0.7 -requests==2.27.1 -requests-toolbelt==1.0.0 -requests-unixsocket==0.3.0 -rfc3986==1.5.0 -Routes==2.5.1 -setuptools==59.6.0 -six==1.16.0 -SQLAlchemy==1.3.20 -statsd==4.0.1 -stevedore==3.5.2 -typing_extensions==4.1.1 -ulid2==0.3.0 -urllib3==1.26.16 -uWSGI==2.0.21 -vine==5.0.0 -WebOb==1.8.7 -wheel==0.37.1 -wrapt==1.15.0 -yappi==1.4.0 -zipp==3.6.0 diff --git a/python/requirements-frozen.txt b/python/requirements-frozen.txt index f99bdedd..6695dc29 100644 --- a/python/requirements-frozen.txt +++ b/python/requirements-frozen.txt @@ -1,131 +1,112 @@ -alabaster==0.7.13 -alembic==1.10.4 +alembic==1.11.1 amqp==5.1.1 attrs==23.1.0 -Babel==2.12.1 -bandit==1.7.5 +backports.zoneinfo==0.2.1 bcrypt==4.0.1 Beaker==1.12.1 -cachetools==5.3.0 -certifi==2022.12.7 +cachetools==5.3.1 +certifi==2023.5.7 cffi==1.15.1 charset-normalizer==3.1.0 -click==6.7 -coverage==7.2.3 +click==8.1.3 cryptography==40.0.2 debtcollector==2.5.0 decorator==5.1.1 defusedxml==0.7.1 dnspython==2.3.0 -docutils==0.19 -dogpile.cache==1.1.8 +dogpile.cache==1.2.2 eventlet==0.33.3 -exceptiongroup==1.1.1 falcon==3.1.1 fasteners==0.18 -fixtures==4.0.1 -flake8==6.0.0 +fixtures==4.1.0 futurist==2.4.1 -gitdb==4.0.10 -GitPython==3.1.31 greenlet==2.0.2 idna==3.4 -imagesize==1.4.1 -iniconfig==2.0.0 -iso8601==1.1.0 +importlib-metadata==4.13.0 +importlib-resources==5.12.0 +iso8601==2.0.0 Jinja2==3.1.2 jsonpatch==1.32 jsonpath-rw==1.4.0 jsonpointer==2.3 -jsonschema==3.2.0 -keystoneauth1==5.1.2 -keystonemiddleware==10.2.0 -kombu==5.2.4 -libvirt-python==9.2.0 +jsonschema==4.17.3 +keystoneauth1==4.3.1 +keystonemiddleware==10.4.0 +kombu==5.3.0 +libvirt-python==9.6.0 Mako==1.2.4 -markdown-it-py==2.2.0 -MarkupSafe==2.1.2 -mccabe==0.7.0 -mdurl==0.1.2 -mock==5.0.1 +MarkupSafe==2.1.3 +mock==5.1.0 msgpack==1.0.5 netaddr==0.8.0 netifaces==0.11.0 oauthlib==3.2.2 os-service-types==1.7.0 -oslo.cache==3.3.1 +oslo.cache==3.4.0 oslo.concurrency==5.1.1 -oslo.config==9.1.1 -oslo.context==5.1.1 +oslo.config==8.6.0 +oslo.context==4.0.0 +oslo.db==11.2.0 oslo.i18n==6.0.0 -oslo.log==5.2.0 -oslo.messaging==14.2.0 +oslo.log==4.8.0 +oslo.messaging==14.3.1 oslo.metrics==0.6.0 oslo.middleware==5.1.1 -oslo.policy==4.1.1 -oslo.serialization==5.1.1 +oslo.policy==3.10.1 +oslo.serialization==4.1.0 oslo.service==3.1.1 -oslo.utils==6.1.0 +oslo.utils==4.12.3 oslo.versionedobjects==3.1.0 -packaging==23.1 -Paste==3.5.2 +packaging==21.3 +Paste==3.5.3 PasteDeploy==3.0.1 pbr==5.11.1 -pip==23.0.1 -pluggy==1.0.0 +pip==23.1.2 +pkgutil_resolve_name==1.3.10 ply==3.11 -prometheus-client==0.16.0 +prometheus-client==0.17.0 psycopg2-binary==2.9.6 PTable==0.9.2 pycadf==3.1.1 -pycodestyle==2.10.0 pycparser==2.21 -pyflakes==3.0.1 -pyghmi==1.5.61 -Pygments==2.15.1 +pyghmi==1.5.62 pylibyaml==0.1.0 -pymongo==3.10.1 +pymongo==4.3.3 pyparsing==3.0.9 pyrsistent==0.19.3 -pytest==7.3.1 -pytest-cov==4.0.0 -pytest-mock==3.10.0 python-dateutil==2.8.2 python-keystoneclient==5.1.0 +python-memcached==1.59 pytz==2023.3 PyYAML==6.0 -redfish==3.1.9 +redfish==3.2.1 repoze.lru==0.7 -requests==2.29.0 -requests-toolbelt==0.10.1 +requests==2.31.0 +requests-toolbelt==1.0.0 requests-unixsocket==0.3.0 responses==0.23.1 -rfc3986==2.0.0 -rich==13.3.4 +rfc3986==1.5.0 Routes==2.5.1 -setuptools==56.0.0 +setuptools==67.7.2 six==1.16.0 -smmap==5.0.0 -snowballstemmer==2.2.0 -Sphinx==6.2.1 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -SQLAlchemy==1.3.21 +SQLAlchemy==1.4.48 +sqlalchemy-migrate==0.13.0 +sqlparse==0.4.4 statsd==4.0.1 -stevedore==5.0.0 -tomli==2.0.1 -types-PyYAML==6.0.12.9 -typing_extensions==4.5.0 -ulid2==0.1.1 -urllib3==1.26.15 -uWSGI==2.0.21 +stevedore==5.1.0 +Tempita==0.5.2 +testresources==2.0.1 +testscenarios==0.5.0 +testtools==2.6.0 +types-PyYAML==6.0.12.10 +typing_extensions==4.6.3 +tzdata==2023.3 +ulid2==0.3.0 +urllib3==1.26.16 +uWSGI==2.0.22 vine==5.0.0 WebOb==1.8.7 wheel==0.40.0 wrapt==1.15.0 -yapf==0.33.0 yappi==1.4.0 +zipp==3.15.0 diff --git a/python/test-requirements.txt b/python/test-requirements.txt index 92e9d790..184b107c 100644 --- a/python/test-requirements.txt +++ b/python/test-requirements.txt @@ -1,15 +1,7 @@ -# tests -click -falcon==3.1.1 -jsonschema==3.2.0 -mock==5.0.1 -pylibyaml==0.1.0 -pymongo pytest >= 3.0 pytest-cov==4.0.0 pytest-mock -responses -ulid2 + # tools yapf diff --git a/python/tests/postgres/start_postgres.sh b/python/tests/postgres/start_postgres.sh index 90443883..ec092708 100755 --- a/python/tests/postgres/start_postgres.sh +++ b/python/tests/postgres/start_postgres.sh @@ -22,11 +22,11 @@ then docker stop 'psql_integration' fi -docker run --rm -dp 5432:5432 --name 'psql_integration' -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.6 +docker run --rm -dp 5432:5432 --name 'psql_integration' -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.8 sleep 15 -docker run --rm --net host postgres:14.6 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres -docker run --rm --net host postgres:14.6 psql -h localhost -c "create database drydock;" postgres postgres +docker run --rm --net host postgres:14.8 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres +docker run --rm --net host postgres:14.8 psql -h localhost -c "create database drydock;" postgres postgres export DRYDOCK_DB_URL="postgresql+psycopg2://drydock:drydock@localhost:5432/drydock" diff --git a/tools/drydock_image_run.sh b/tools/drydock_image_run.sh index ddae3c34..c872bee8 100755 --- a/tools/drydock_image_run.sh +++ b/tools/drydock_image_run.sh @@ -11,11 +11,11 @@ function start_db { sudo docker stop "${PSQL_CONTAINER_NAME}" fi - docker run --rm -dp 5432:5432 --name "${PSQL_CONTAINER_NAME}" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.6 + docker run --rm -dp 5432:5432 --name "${PSQL_CONTAINER_NAME}" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.8 sleep 15 - docker run --rm --net host postgres:14.6 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres - docker run --rm --net host postgres:14.6 psql -h localhost -c "create database drydock;" postgres postgres + docker run --rm --net host postgres:14.8 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres + docker run --rm --net host postgres:14.8 psql -h localhost -c "create database drydock;" postgres postgres } function customize_conf { diff --git a/tools/helm_install.sh b/tools/helm_install.sh index fa116acc..81eddd2b 100755 --- a/tools/helm_install.sh +++ b/tools/helm_install.sh @@ -17,7 +17,7 @@ set -x HELM=$1 -HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz"} +HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"} function install_helm_binary { diff --git a/tox.ini b/tox.ini index 982e9895..95672430 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ setenv = YAMLDIR = {toxinidir}/python/tests/yaml_samples/ passenv = http_proxy,HTTP_PROXY,https_proxy,HTTPS_PROXY,no_proxy,NO_PROXY deps= -r{toxinidir}/python/requirements-frozen.txt + -r{toxinidir}/python/test-requirements.txt [testenv:venv] commands = {posargs} @@ -19,26 +20,11 @@ allowlist_externals= sh deps= -r{toxinidir}/python/requirements-direct.txt - -r{toxinidir}/python/test-requirements.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt commands= rm -f python/requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'drydock_provisioner|pyinotify|pkg-resources==0.0.0' > python/requirements-frozen.txt" - -[testenv:freeze-bionic] -basepython=python3 -recreate = True -allowlist_externals= - rm - sh -deps= - -r{toxinidir}/python/requirements-direct.txt -commands= - rm -f python/requirements-frozen-bionic.txt - sh -c "pip freeze --all | grep -vE 'drydock_provisioner|pyinotify|pkg-resources==0.0.0' > python/requirements-frozen-bionic.txt" - - - [testenv:yapf] allowlist_externals=find commands= @@ -50,6 +36,8 @@ commands= [testenv:py38] usedevelop=True +allowlist_externals= + py.test setenv= PYTHONWARNING=all YAMLDIR={toxinidir}/python/tests/yaml_samples/ @@ -58,21 +46,6 @@ commands= --cov=drydock_provisioner \ {toxinidir}/python/tests/unit/{posargs} -[testenv:py36] -usedevelop=True -setenv= - PYTHONWARNING=all - YAMLDIR={toxinidir}/python/tests/yaml_samples/ -deps= - -r{toxinidir}/python/requirements-frozen-bionic.txt - -r{toxinidir}/python/test-requirements.txt - tox -commands= - py.test \ - --cov=drydock_provisioner \ - {toxinidir}/python/tests/unit/{posargs} - - [testenv:integration] passenv=DOCKER_REGISTRY,IMAGE_NAME,IMAGE_PREFIX,IMAGE_TAG,DISTRO setenv= @@ -81,6 +54,7 @@ setenv= allowlist_externals= sh pip3 + py.test commands= pip3 install ./python/ --use-pep517 sh -c {toxinidir}/python/tests/postgres/start_postgres.sh