From 37ba567da5bf7f146f54a468478a025c70793574 Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Sat, 1 Jul 2023 02:08:39 +0000 Subject: [PATCH] Airflow stable 2.6.2 This PS updates python modules and code to match Airflow 2.6.2: - bionic py36 gates were removed - python code corrected to match new modules versions - selection of python modules versions was perfrmed based on airflow-2.6.2 constraints - airskiff deploy pipeline was aligned with latest in treasuremap v1.9 - postgresql image updated to 14.8 Change-Id: I65a1b86473ee3e988aae353b59fb5473d75851f9 --- .zuul.yaml | 236 ++++-------------- ChangeLog | 3 + deckhand/conf/config.py | 6 +- deckhand/tests/common/test_gabbi.py | 2 +- entrypoint.sh | 6 +- etc/deckhand/deckhand.conf.sample | 11 +- requirements-direct.txt | 89 ++----- requirements-frozen-bionic.txt | 156 ------------ requirements-frozen.txt | 152 +++++------ test-requirements.txt | 30 +-- tools/gate/playbooks/airskiff-deploy.yaml | 29 ++- .../gate/playbooks/airskiff-reduce-site.yaml | 1 + .../run-integration-tests-docker.yaml | 2 +- .../tasks/deploy-keystone-dependencies.yaml | 33 ++- .../tasks/install-test-requirements.yaml | 15 +- .../tasks/integration-tests.yaml | 11 +- tools/integration-tests.sh | 1 + tox.ini | 81 +----- 18 files changed, 218 insertions(+), 646 deletions(-) delete mode 100644 requirements-frozen-bionic.txt diff --git a/.zuul.yaml b/.zuul.yaml index e0ed510d..e144d783 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -12,57 +12,42 @@ - project: templates: - - openstack-cover-jobs - docs-on-readthedocs - openstack-python38-jobs - - openstack-python36-jobs vars: rtd_webhook_id: '38572' rtd_project_name: 'airship-deckhand' check: jobs: - - deckhand-tox-py36-postgresql - deckhand-tox-py38-postgresql - - deckhand-functional-uwsgi-py36 - deckhand-functional-uwsgi-py38 - - deckhand-functional-docker-py38-ubuntu_focal - - deckhand-functional-docker-py36-ubuntu_bionic - - deckhand-integration-uwsgi-py36 + - deckhand-functional-docker-py38 - deckhand-integration-uwsgi-py38 - - deckhand-integration-docker-py38-ubuntu_focal - - deckhand-integration-docker-py36-ubuntu_bionic + - deckhand-integration-docker-py38 - deckhand-chart-build-gate - deckhand-chart-build-latest-htk - - deckhand-docker-build-gate-ubuntu_focal - - deckhand-docker-build-gate-ubuntu_bionic - - openstack-tox-pep8 + - deckhand-docker-build-gate + - deckhand-openstack-tox-pep8-focal + - deckhand-openstack-tox-cover-focal - deckhand-airskiff-deployment-focal - - deckhand-airskiff-deployment-bionic + gate: jobs: - - deckhand-tox-py36-postgresql - deckhand-tox-py38-postgresql - - deckhand-functional-uwsgi-py36 - deckhand-functional-uwsgi-py38 - - deckhand-functional-docker-py38-ubuntu_focal - - deckhand-functional-docker-py36-ubuntu_bionic - - deckhand-integration-uwsgi-py36 + - deckhand-functional-docker-py38 - deckhand-integration-uwsgi-py38 - - deckhand-integration-docker-py38-ubuntu_focal - - deckhand-integration-docker-py36-ubuntu_bionic + - deckhand-integration-docker-py38 - deckhand-chart-build-gate - - deckhand-docker-build-gate-ubuntu_focal - - deckhand-docker-build-gate-ubuntu_bionic - - openstack-tox-pep8 + - deckhand-docker-build-gate + - deckhand-openstack-tox-pep8-focal + - deckhand-openstack-tox-cover-focal - deckhand-airskiff-deployment-focal - - deckhand-airskiff-deployment-bionic + post: jobs: - deckhand-upload-git-mirror - deckhand-docker-publish-ubuntu_focal - # - deckhand-docker-tag-ubuntu_focal - - deckhand-docker-publish-ubuntu_bionic - # - deckhand-docker-tag-ubuntu_bionic - nodeset: @@ -89,6 +74,18 @@ - name: primary label: ubuntu-focal +- job: + name: deckhand-openstack-tox-pep8-focal + parent: openstack-tox-pep8 + description: Runs pep8 job on focal + nodeset: deckhand-single-node-airskiff-focal + +- job: + name: deckhand-openstack-tox-cover-focal + parent: openstack-tox-cover + description: Runs cover job on focal + nodeset: deckhand-single-node-airskiff-focal + - job: name: deckhand-tox-py38-postgresql parent: openstack-tox-py38 @@ -98,35 +95,6 @@ vars: tox_envlist: py38-postgresql -- job: - name: deckhand-tox-py36-postgresql - parent: openstack-tox-py38 - nodeset: deckhand-single-node - pre-run: - - tools/gate/playbooks/install-postgresql.yaml - vars: - tox_envlist: py36-postgresql - -- job: - name: deckhand-functional-uwsgi-py36 - voting: true - description: | - Run tox-based functional tests for the Airship Deckhand project using a - minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf - for ephemeral PostgreSQL DB, under cPython version 3.6. - run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml - nodeset: deckhand-single-node - vars: - tox_envlist: py36-functional-dev - distro: ubuntu_bionic - irrelevant-files: &irrelevant-files - - ^.*\.rst$ - - ^doc/.*$ - - ^etc/.*$ - - ^releasenotes/.*$ - - ^setup.cfg$ - - ^deckhand/tests/unit/.*$ - - job: name: deckhand-functional-uwsgi-py38 @@ -140,7 +108,13 @@ vars: tox_envlist: py38-functional-dev distro: ubuntu_focal - irrelevant-files: *irrelevant-files + irrelevant-files: &irrelevant-files + - ^.*\.rst$ + - ^doc/.*$ + - ^etc/.*$ + - ^releasenotes/.*$ + - ^setup.cfg$ + - ^deckhand/tests/unit/.*$ - job: name: deckhand-functional-docker-base @@ -159,7 +133,7 @@ - openstack/openstack-helm-infra - job: - name: deckhand-functional-docker-py38-ubuntu_focal + name: deckhand-functional-docker-py38 description: | Run tox-based functional tests for the Airship Deckhand project under cPython version 3.8. Uses tox with the ``functional-py38`` environment. @@ -172,36 +146,6 @@ distro: ubuntu_focal irrelevant-files: *irrelevant-files -- job: - name: deckhand-functional-docker-py36-ubuntu_bionic - description: | - Run tox-based functional tests for the Airship Deckhand project under - cPython version 3.6. Uses tox with the ``functional-py36`` environment. - Ubuntu (focal) image is built and used. - parent: deckhand-functional-docker-base - nodeset: deckhand-single-node - vars: - tox_envlist: py36-functional - disable_keystone: true - distro: ubuntu_bionic - irrelevant-files: *irrelevant-files - - - -- job: - name: deckhand-integration-uwsgi-py36 - voting: true - description: | - Run tox-based integration tests for the Airship Deckhand project using a - minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf - for ephemeral PostgreSQL DB, under cPython version 3.6. - timeout: 3600 - run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml - nodeset: deckhand-single-node - irrelevant-files: *irrelevant-files - vars: - disable_keystone: true - distro: ubuntu_bionic - job: name: deckhand-integration-uwsgi-py38 @@ -210,7 +154,7 @@ Run tox-based integration tests for the Airship Deckhand project using a minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf for ephemeral PostgreSQL DB, under cPython version 3.8. - timeout: 3600 + timeout: 9600 run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml nodeset: deckhand-single-node-focal irrelevant-files: *irrelevant-files @@ -248,6 +192,7 @@ vars: zuul_osh_relative_path: ../../openstack/openstack-helm/ zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra/ + zuul_treasuremap_relative_path: ../../airship/treasuremap/ pre-run: - tools/gate/playbooks/osh-infra-upgrade-host.yaml - tools/gate/playbooks/osh-infra-deploy-docker.yaml @@ -257,13 +202,14 @@ required-projects: - openstack/openstack-helm - openstack/openstack-helm-infra + - airship/treasuremap irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - ^releasenotes/.*$ - job: - name: deckhand-integration-docker-py38-ubuntu_focal + name: deckhand-integration-docker-py38 description: | Run tox-based integration tests for the Airship Deckhand project under cPython version 3.8. Builds ubuntu (focal) deckhand image. @@ -273,17 +219,6 @@ disable_keystone: false distro: ubuntu_focal -- job: - name: deckhand-integration-docker-py36-ubuntu_bionic - description: | - Run tox-based integration tests for the Airship Deckhand project under - cPython version 3.6. Builds ubuntu (focal) deckhand image. - parent: deckhand-integration-docker-base - nodeset: openstack-helm-single-node - vars: - disable_keystone: false - distro: ubuntu_focal - - job: name: deckhand-airskiff-deployment-focal voting: true @@ -292,8 +227,8 @@ Deploy Memcached using Airskiff and submitted Deckhand changes. timeout: 9600 pre-run: - - tools/gate/playbooks/airskiff-reduce-site.yaml - tools/gate/playbooks/git-config.yaml + - tools/gate/playbooks/airskiff-reduce-site.yaml run: tools/gate/playbooks/airskiff-deploy.yaml post-run: tools/gate/playbooks/debug-report.yaml required-projects: @@ -308,33 +243,9 @@ - ^doc/.*$ - ^releasenotes/.*$ -- job: - name: deckhand-airskiff-deployment-bionic - voting: true - nodeset: deckhand-single-node-airskiff-focal - description: | - Deploy Memcached using Airskiff and submitted Deckhand changes. - timeout: 9600 - pre-run: - - tools/gate/playbooks/airskiff-reduce-site.yaml - - tools/gate/playbooks/git-config.yaml - run: tools/gate/playbooks/airskiff-deploy.yaml - post-run: tools/gate/playbooks/debug-report.yaml - required-projects: - - airship/treasuremap - vars: - CLONE_DECKHAND: false - OSH_INFRA_COMMIT: 443ff3e3e340c94c5cbb214d1e2a8b2a3937541d - DECKHAND_IMAGE_DISTRO: ubuntu_bionic - distro: ubuntu_focal - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - ^releasenotes/.*$ - - job: - name: deckhand-docker-build-gate-ubuntu_focal + name: deckhand-docker-build-gate timeout: 3600 run: tools/gate/playbooks/docker-image-build.yaml nodeset: deckhand-single-node-focal @@ -352,19 +263,6 @@ dynamic: patch_set: true -- job: - name: deckhand-docker-build-gate-ubuntu_bionic - timeout: 3600 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node - irrelevant-files: *non-code-files-template - vars: - publish: false - distro: ubuntu_bionic - tags: - dynamic: - patch_set: true - - job: name: deckhand-docker-publish-ubuntu_focal description: | @@ -386,61 +284,23 @@ commit: true static: - latest + - airflow_2.6.2 - job: - name: deckhand-docker-publish-ubuntu_bionic + name: deckhand-docker-tag-ubuntu_focal description: | - Runs on every merge, unless files in a dictionary below are changed. - Builds and publishes container ubuntu images on quay.io with a set of tags - listed in vars section. Waits in Zuul queue for a node (VM) assignment. + Runs on every merge when files in a dictionalry below are changed, and + adds git commit id tag onto the ubuntu container image published on quay.io, + which has `latest` tag set. Does not wait in queue for a node (VM) + assignment, runs almost immediately. timeout: 3600 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node + run: tools/gate/playbooks/docker-image-tag.yaml + nodeset: + nodes: [] secrets: - airship_deckhand_quay_creds - irrelevant-files: *non-code-files-template vars: - publish: true - distro: ubuntu_bionic - tags: - dynamic: - branch: true - commit: true - static: - - latest - -# - job: -# name: deckhand-docker-tag-ubuntu_focal -# description: | -# Runs on every merge when files in a dictionalry below are changed, and -# adds git commit id tag onto the ubuntu container image published on quay.io, -# which has `latest` tag set. Does not wait in queue for a node (VM) -# assignment, runs almost immediately. -# timeout: 3600 -# run: tools/gate/playbooks/docker-image-tag.yaml -# nodeset: -# nodes: [] -# secrets: -# - airship_deckhand_quay_creds -# vars: -# distro: ubuntu_focal - -# - job: -# name: deckhand-docker-tag-ubuntu_bionic -# description: | -# Runs on every merge when files in a dictionalry below are changed, and -# adds git commit id tag onto the ubuntu container image published on quay.io, -# which has `latest` tag set. Does not wait in queue for a node (VM) -# assignment, runs almost immediately. -# timeout: 3600 -# run: tools/gate/playbooks/docker-image-tag.yaml -# nodeset: -# nodes: [] -# secrets: -# - airship_deckhand_quay_creds -# vars: -# distro: ubuntu_bionic - + distro: ubuntu_focal - secret: diff --git a/ChangeLog b/ChangeLog index ef7422a8..7cedf51a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,9 @@ CHANGES ======= +* Airflow stable 2.6.2 +* Restored ubuntu\_bionic image build +* [focal] Fix requests.body attribute deprecation * Update airskiff deployment gate * Deckhand updates * Removing egg-info folder diff --git a/deckhand/conf/config.py b/deckhand/conf/config.py index f405d38f..9d9de70c 100644 --- a/deckhand/conf/config.py +++ b/deckhand/conf/config.py @@ -43,7 +43,7 @@ barbican_opts = [ help="Whether to enable Barbican secret caching. Useful " "for testing to avoid cross-test caching conflicts."), cfg.StrOpt( - 'cache_timeout', default=3600, + 'cache_timeout', default='3600', help="How long (in seconds) Barbican secret reference/payload lookup " "results should remain cached in memory.") ] @@ -63,7 +63,7 @@ engine_opts = [ cfg.BoolOpt('enable_cache', default=True, help="Whether to enable the document rendering caching. Useful" " for testing to avoid cross-test caching conflicts."), - cfg.IntOpt('cache_timeout', default=3600, + cfg.IntOpt('cache_timeout', default='3600', help="How long (in seconds) document rendering results should " "remain cached in memory."), ] @@ -76,7 +76,7 @@ jsonpath_group = cfg.OptGroup( jsonpath_opts = [ - cfg.IntOpt('cache_timeout', default=3600, + cfg.IntOpt('cache_timeout', default='3600', help="How long (in seconds) JSONPath lookup results should " "remain cached in memory.") ] diff --git a/deckhand/tests/common/test_gabbi.py b/deckhand/tests/common/test_gabbi.py index 647fc780..6624671d 100644 --- a/deckhand/tests/common/test_gabbi.py +++ b/deckhand/tests/common/test_gabbi.py @@ -90,7 +90,7 @@ class MultidocJsonpaths(jsonhandler.JSONHandler): # NOTE: The simple approach to handling dictionary versus list response # bodies is to always parse the response body as a list and index into # the first element using [0] throughout the tests. - return list(yaml.load_all(string)) + return list(yaml.safe_load_all(string)) def load_tests(loader, tests, pattern): diff --git a/entrypoint.sh b/entrypoint.sh index 2f43b2d4..0751be80 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -38,13 +38,13 @@ echo "Command: $1 with arguments $@" # Start deckhand application if [ "$1" = 'server' ]; then exec uwsgi \ - -b 32768 \ + --buffer-size 32768 \ --callable deckhand_callable \ --die-on-term \ --enable-threads \ --http :${PORT} \ --http-timeout ${DECKHAND_API_TIMEOUT} \ - -L \ + --disable-logging \ --lazy-apps \ --master \ --pyargv "--config-file ${DECKHAND_CONFIG_DIR}/deckhand.conf" \ @@ -53,7 +53,7 @@ if [ "$1" = 'server' ]; then --strict \ --vacuum \ --need-app \ - -w deckhand.cmd + --module deckhand.cmd elif [ "$1" = 'alembic' ]; then exec alembic ${@:2} else diff --git a/etc/deckhand/deckhand.conf.sample b/etc/deckhand/deckhand.conf.sample index 29bb5d52..39680e8a 100644 --- a/etc/deckhand/deckhand.conf.sample +++ b/etc/deckhand/deckhand.conf.sample @@ -114,7 +114,7 @@ # Format string to use for log messages with context. Used by # oslo_log.formatters.ContextFormatter (string value) -#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s +#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s # Format string to use for log messages when context is undefined. Used by # oslo_log.formatters.ContextFormatter (string value) @@ -131,7 +131,7 @@ # Defines the format string for %(user_identity)s that is used in # logging_context_format_string. Used by oslo_log.formatters.ContextFormatter # (string value) -#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s +#logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s # List of package logging levels in logger=LEVEL pairs. This option is ignored # if log_config_append is set. (list value) @@ -242,7 +242,7 @@ # Domain name containing project (string value) #project_domain_name = -# ID of the trust to use as a trustee use (string value) +# Trust ID (string value) #trust_id = # User ID (string value) @@ -508,7 +508,7 @@ # Domain name containing project (string value) #project_domain_name = -# ID of the trust to use as a trustee use (string value) +# Trust ID (string value) #trust_id = # Optional domain ID to use with v3 and v2 parameters. It will be used for both @@ -718,6 +718,9 @@ # middleware should parse the headers or not. (boolean value) #enable_proxy_headers_parsing = false +# HTTP basic auth password file. (string value) +#http_basic_auth_user_file = /etc/htpasswd + [oslo_policy] diff --git a/requirements-direct.txt b/requirements-direct.txt index f21b04cb..85ae52e1 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -1,77 +1,28 @@ -alembic<=1.4.3 -amqp==2.6.1 -Beaker<=1.12.0 -chardet==3.0.4 -charset_normalizer >=2.0.0, < 4.0.0 -ConfigParser -coverage -cryptography==3.4.8 -deepdiff<=5.8.1 -docutils<=0.17 +barbican==14.0.2 +Beaker +deepdiff falcon -hacking -importlib-metadata~=2.0 -Jinja2<=3.0.3 +fixtures jsonpath_ng -jsonpath-rw-ext>=1.0.0 -jsonpickle==1.4.1 -jsonschema<=3.2.0 -keystoneauth1<=5.1.1 -keystonemiddleware -kombu<=4.6.11 -MarkupSafe<2.1.0, >=0.9.2 +jsonpickle +jsonschema +keystoneauth1==4.3.1 networkx -nose -oslo.cache<=2.10.1 -oslo.concurrency -oslo.config<=8.7.1 -oslo.context<=4.1.0 -oslo.db<=10.0.0 -oslo.log<=4.6.0 -oslo.messaging<=12.13.0 -oslo.middleware<=4.4.0 -oslo.policy<=3.10.1 -oslo.serialization<=4.2.0 -oslo.utils<=4.12.3 -packaging==21.3 -Paste<=3.5.0 -PasteDeploy +oslo.config==8.6.0 +oslo.context==4.0.0 +oslo.log==4.8.0 +oslo.policy==3.10.1 +oslo.serialization==4.1.0 +oslo.utils==4.12.3 +oslo.db==11.2.0 PasteScript -pbr<=5.5.1 -psycopg2-binary -pycadf<=3.1.1 -pycodestyle<=2.6.0 -pyflakes<=2.2.0 -Pygments<=2.14.0 -pylibyaml==0.1.0 -pymongo -pyparsing<=2.4.7 -pyproject-api -python-barbicanclient<=5.2.0 -python-dateutil==2.8.1 -python-keystoneclient +pylibyaml +python-barbicanclient python-memcached -python-subunit<=1.4.0 -PyYAML<=5.4.1 -reno -requests==2.23.0 -Routes -setuptools<=45.2.0 +PyYAML six -Sphinx==3.3.1 -sphinx-rtd-theme==0.5.0 -SQLAlchemy<=1.3.20 -stevedore -testrepository -testresources -testscenarios -testtools<=2.5.0 +testtools tiddlyweb -typing-extensions==3.7.4.3 -urllib3 >= 1.21.1, <= 1.25.11 -uWSGI==2.0.21 -virtualenv Werkzeug -wheel -wsgi-intercept>=1.2.2 -yq==3.2.1 +psycopg2-binary +uwsgi \ No newline at end of file diff --git a/requirements-frozen-bionic.txt b/requirements-frozen-bionic.txt deleted file mode 100644 index 8aa45bda..00000000 --- a/requirements-frozen-bionic.txt +++ /dev/null @@ -1,156 +0,0 @@ -alabaster==0.7.13 -alembic==1.4.3 -amqp==2.6.1 -argcomplete==3.0.8 -attrs==22.2.0 -autopage==0.5.1 -Babel==2.11.0 -bcrypt==4.0.1 -Beaker==1.12.0 -cachetools==4.2.4 -certifi==2023.5.7 -cffi==1.15.1 -chardet==3.0.4 -charset-normalizer==3.0.1 -cliff==3.10.1 -cmd2==2.4.3 -configparser==5.2.0 -coverage==6.2 -cryptography==3.4.8 -dataclasses==0.8 -debtcollector==2.5.0 -decorator==4.4.2 -deepdiff==5.7.0 -distlib==0.3.6 -dnspython==2.2.1 -docutils==0.17 -dogpile.cache==1.2.0 -dulwich==0.20.50 -eventlet==0.33.3 -extras==1.0.0 -falcon==3.1.1 -fasteners==0.18 -filelock==3.4.1 -fixtures==3.0.0 -flake8==3.8.4 -futurist==2.4.1 -greenlet==2.0.2 -hacking==4.1.0 -html5lib==0.9999999 -httpexceptor==1.4.0 -idna==2.10 -imagesize==1.4.1 -importlib-metadata==2.1.3 -importlib-resources==5.4.0 -iso8601==1.1.0 -Jinja2==3.0.3 -jsonpath-ng==1.5.3 -jsonpath-rw==1.4.0 -jsonpath-rw-ext==1.2.2 -jsonpickle==1.4.1 -jsonschema==3.2.0 -keystoneauth1==5.1.1 -keystonemiddleware==9.5.0 -kombu==4.6.11 -Mako==1.1.6 -MarkupSafe==2.0.1 -mccabe==0.6.1 -msgpack==1.0.5 -netaddr==0.8.0 -netifaces==0.11.0 -networkx==2.5.1 -nose==1.3.7 -ordered-set==4.0.2 -os-service-types==1.7.0 -oslo.cache==2.10.1 -oslo.concurrency==4.5.1 -oslo.config==8.7.1 -oslo.context==4.1.0 -oslo.db==10.0.0 -oslo.i18n==5.1.0 -oslo.log==4.6.0 -oslo.messaging==12.13.0 -oslo.metrics==0.4.0 -oslo.middleware==4.4.0 -oslo.policy==3.10.1 -oslo.serialization==4.2.0 -oslo.service==2.8.0 -oslo.utils==4.12.3 -packaging==21.3 -Paste==3.5.0 -PasteDeploy==2.1.1 -PasteScript==3.3.0 -pbr==5.5.1 -pip==21.3.1 -platformdirs==2.4.0 -ply==3.11 -prettytable==2.5.0 -prometheus-client==0.16.0 -psycopg2-binary==2.9.6 -pycadf==3.1.1 -pycodestyle==2.6.0 -pycparser==2.21 -pyflakes==2.2.0 -Pygments==2.14.0 -pylibyaml==0.1.0 -pymongo==4.1.1 -pyparsing==2.4.7 -pyperclip==1.8.2 -pyproject-api==0.1.0 -pyrsistent==0.18.0 -python-barbicanclient==5.2.0 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-keystoneclient==4.5.0 -python-memcached==1.59 -python-mimeparse==1.6.0 -python-subunit==1.4.0 -pytz==2023.3 -PyYAML==5.4.1 -reno==4.0.0 -repoze.lru==0.7 -requests==2.23.0 -resolver==0.2.1 -rfc3986==1.5.0 -Routes==2.5.1 -selector==0.10.1 -setuptools==45.2.0 -simplejson==3.19.1 -six==1.16.0 -snowballstemmer==2.2.0 -Sphinx==3.3.1 -sphinx-rtd-theme==0.5.0 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.0 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -SQLAlchemy==1.3.20 -sqlalchemy-migrate==0.13.0 -sqlparse==0.4.4 -statsd==4.0.1 -stevedore==3.5.2 -Tempita==0.5.2 -testrepository==0.0.20 -testresources==2.0.1 -testscenarios==0.5.0 -testtools==2.5.0 -tiddlyweb==2.4.3 -tomli==1.2.3 -tomlkit==0.11.6 -typing-extensions==3.7.4.3 -urllib3==1.25.11 -uWSGI==2.0.21 -vine==1.3.0 -virtualenv==20.16.2 -wcwidth==0.2.6 -WebOb==1.8.7 -Werkzeug==2.0.3 -wheel==0.37.1 -wrapt==1.15.0 -wsgi-intercept==1.11.0 -xmltodict==0.13.0 -yappi==1.4.0 -yq==3.2.1 -zipp==3.6.0 diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 436313cf..fcacec10 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,154 +1,124 @@ -alabaster==0.7.13 -alembic==1.4.3 -amqp==2.6.1 -argcomplete==3.0.8 +alembic==1.11.1 +amqp==5.1.1 attrs==23.1.0 autopage==0.5.1 -Babel==2.12.1 +backports.zoneinfo==0.2.1 +barbican==14.0.2 bcrypt==4.0.1 -Beaker==1.12.0 -cachetools==5.3.0 -certifi==2022.12.7 +Beaker==1.12.1 +cachetools==5.3.1 +castellan==4.1.0 +certifi==2023.5.7 cffi==1.15.1 -chardet==3.0.4 charset-normalizer==3.1.0 -cliff==3.10.1 +cliff==4.3.0 cmd2==2.4.3 -configparser==5.3.0 -coverage==7.2.3 -cryptography==3.4.8 +cryptography==40.0.2 debtcollector==2.5.0 decorator==5.1.1 -deepdiff==5.8.1 -distlib==0.3.6 +deepdiff==6.3.1 dnspython==2.3.0 -docutils==0.17 -dogpile.cache==1.1.8 -dulwich==0.21.3 +dogpile.cache==1.2.2 eventlet==0.33.3 -extras==1.0.0 falcon==3.1.1 fasteners==0.18 -filelock==3.12.0 -fixtures==3.0.0 -flake8==3.8.4 +fixtures==4.1.0 futurist==2.4.1 greenlet==2.0.2 -hacking==4.1.0 html5lib==0.9999999 httpexceptor==1.4.0 -idna==2.10 -imagesize==1.4.1 -importlib-metadata==2.1.3 -iso8601==1.1.0 -Jinja2==3.0.3 +idna==3.4 +importlib-metadata==4.13.0 +importlib-resources==5.12.0 +iso8601==2.0.0 +Jinja2==3.1.2 jsonpath-ng==1.5.3 -jsonpath-rw==1.4.0 -jsonpath-rw-ext==1.2.2 -jsonpickle==1.4.1 -jsonschema==3.2.0 -keystoneauth1==5.1.1 -keystonemiddleware==10.2.0 -kombu==4.6.11 +jsonpickle==3.0.1 +jsonschema==4.17.3 +keystoneauth1==4.3.1 +keystonemiddleware==10.4.0 +kombu==5.3.0 +ldap3==2.9.1 +logutils==0.3.5 Mako==1.2.4 -MarkupSafe==2.0.1 -mccabe==0.6.1 +MarkupSafe==2.1.3 msgpack==1.0.5 netaddr==0.8.0 netifaces==0.11.0 networkx==3.1 -nose==1.3.7 ordered-set==4.1.0 os-service-types==1.7.0 -oslo.cache==2.10.1 +oslo.cache==3.4.0 oslo.concurrency==5.1.1 -oslo.config==8.7.1 -oslo.context==4.1.0 -oslo.db==10.0.0 +oslo.config==8.6.0 +oslo.context==4.0.0 +oslo.db==11.2.0 oslo.i18n==6.0.0 -oslo.log==4.6.0 -oslo.messaging==12.13.0 +oslo.log==4.8.0 +oslo.messaging==14.3.1 oslo.metrics==0.6.0 -oslo.middleware==4.4.0 +oslo.middleware==5.1.1 oslo.policy==3.10.1 -oslo.serialization==4.2.0 +oslo.serialization==4.1.0 oslo.service==3.1.1 +oslo.upgradecheck==2.1.1 oslo.utils==4.12.3 +oslo.versionedobjects==3.1.0 packaging==21.3 -Paste==3.5.0 +Paste==3.5.3 PasteDeploy==3.0.1 PasteScript==3.3.0 -pbr==5.5.1 -pip==23.0.1 -platformdirs==3.2.0 +pbr==5.11.1 +pecan==1.5.1 +pip==23.1.2 +pkgutil_resolve_name==1.3.10 ply==3.11 -prettytable==3.7.0 -prometheus-client==0.16.0 +prettytable==3.8.0 +prometheus-client==0.17.0 psycopg2-binary==2.9.6 +pyasn1==0.4.8 pycadf==3.1.1 -pycodestyle==2.6.0 pycparser==2.21 -pyflakes==2.2.0 -Pygments==2.14.0 pylibyaml==0.1.0 -pymongo==4.3.3 -pyparsing==2.4.7 +pyOpenSSL==23.2.0 +pyparsing==3.0.9 pyperclip==1.8.2 -pyproject_api==1.5.0 pyrsistent==0.19.3 -python-barbicanclient==5.2.0 -python-dateutil==2.8.1 -python-editor==1.0.4 +python-barbicanclient==5.4.0 +python-dateutil==2.8.2 python-keystoneclient==5.1.0 python-memcached==1.59 python-mimeparse==1.6.0 -python-subunit==1.4.0 pytz==2023.3 -PyYAML==5.4.1 -reno==4.0.0 +PyYAML==6.0 repoze.lru==0.7 -requests==2.23.0 +requests==2.31.0 resolver==0.2.1 -rfc3986==2.0.0 +rfc3986==1.5.0 Routes==2.5.1 selector==0.10.1 -setuptools==45.2.0 +setuptools==67.7.2 simplejson==3.19.1 six==1.16.0 -snowballstemmer==2.2.0 -Sphinx==3.3.1 -sphinx-rtd-theme==0.5.0 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -SQLAlchemy==1.3.20 +SQLAlchemy==1.4.48 sqlalchemy-migrate==0.13.0 sqlparse==0.4.4 statsd==4.0.1 -stevedore==5.0.0 +stevedore==5.1.0 Tempita==0.5.2 -testrepository==0.0.20 testresources==2.0.1 testscenarios==0.5.0 -testtools==2.5.0 +testtools==2.6.0 tiddlyweb==2.4.3 -tomli==2.0.1 -tomlkit==0.11.7 -typing-extensions==3.7.4.3 -urllib3==1.25.11 -uWSGI==2.0.21 -vine==1.3.0 -virtualenv==20.22.0 +typing_extensions==4.6.3 +tzdata==2023.3 +urllib3==1.26.16 +uWSGI==2.0.22 +vine==5.0.0 wcwidth==0.2.6 WebOb==1.8.7 -Werkzeug==2.1.2 +Werkzeug==2.2.3 wheel==0.40.0 wrapt==1.15.0 -wsgi-intercept==1.11.0 -xmltodict==0.13.0 yappi==1.4.0 -yq==3.2.1 zipp==3.15.0 diff --git a/test-requirements.txt b/test-requirements.txt index 4d573831..a664112d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,29 +1,29 @@ -amqp +# amqp bandit==1.6.0 -chardet==3.0.4 -fixtures==3.0.0 +# chardet==3.0.4 +# fixtures==3.0.0 flake8==3.8.4 gabbi==1.35.1 -jsonpickle==1.4.1 +# jsonpickle==1.4.1 +# openstacksdk==0.36.5 openstacksdk==0.36.5 -openstacksdk==0.36.5 -os-testr==2.0.1 +# os-testr==2.0.1 os-testr==2.0.1 oslotest==4.5.0 -packaging==21.3 +# packaging==21.3 pifpaf==3.1.5 -pylibyaml==0.1.0 -pyparsing==2.4.7 +# pylibyaml==0.1.0 +# pyparsing==2.4.7 pytest >= 3.0 -pytest-cov==4.0.0 +pytest-cov python-subunit==1.4.0 -requests==2.23.0 -six==1.16.0 +# requests==2.23.0 +# six==1.16.0 stestr==3.2.0 testrepository==0.0.20 -testtools==2.5.0 +# testtools==2.5.0 tox>=3.28.0, <4.0.0 -urllib3==1.25.11 -yq==3.2.1 \ No newline at end of file +# urllib3==1.25.11 +# yq==3.2.1 \ No newline at end of file diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml index e9425622..cd9194c6 100644 --- a/tools/gate/playbooks/airskiff-deploy.yaml +++ b/tools/gate/playbooks/airskiff-deploy.yaml @@ -17,6 +17,8 @@ - clear-firewall - bindep - ensure-docker + - ensure-python + - ensure-pip - disable-systemd-resolved - install-test-requirements @@ -31,6 +33,7 @@ - name: Clone Required Repositories shell: | + set -ex export CLONE_DECKHAND={{ CLONE_DECKHAND }} export OSH_INFRA_COMMIT={{ OSH_INFRA_COMMIT }} ./tools/deployment/airskiff/developer/000-clone-dependencies.sh @@ -60,6 +63,14 @@ args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + - name: Setup openstack client + shell: | + set -ex + ./tools/deployment/airskiff/developer/020-setup-client.sh + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + - name: Build Deckhand with submitted changes shell: | set -ex @@ -76,18 +87,10 @@ export DISTRO={{ DECKHAND_IMAGE_DISTRO }} docker rm registry --force || true docker run -d -p 5000:5000 --restart=always --name registry registry:2 - if test "${DISTRO}" = 'ubuntu_bionic' - then - docker tag quay.io/airshipit/deckhand:latest-ubuntu_bionic localhost:5000/deckhand:latest-ubuntu_bionic - docker push localhost:5000/deckhand:latest-ubuntu_bionic - sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_bionic#g" ./site/airskiff/software/config/versions.yaml - sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_bionic#g" ./global/software/config/versions.yaml - else - docker tag quay.io/airshipit/deckhand:latest-ubuntu_focal localhost:5000/deckhand:latest-ubuntu_focal - docker push localhost:5000/deckhand:latest-ubuntu_focal - sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml - sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_focal#g" ./global/software/config/versions.yaml - fi + docker tag quay.io/airshipit/deckhand:latest-ubuntu_focal localhost:5000/deckhand:latest-ubuntu_focal + docker push localhost:5000/deckhand:latest-ubuntu_focal + sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml + sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_focal#g" ./global/software/config/versions.yaml args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" become: yes @@ -116,6 +119,7 @@ shell: | mkdir -p ~/.kube cp -rp /home/zuul/.kube/config ~/.kube/config + pip3 install packaging ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" @@ -130,6 +134,7 @@ - name: Wait for deployment completion shell: | + set -x ./tools/gate/wait-for-shipyard.sh args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" diff --git a/tools/gate/playbooks/airskiff-reduce-site.yaml b/tools/gate/playbooks/airskiff-reduce-site.yaml index 7c80ed77..2fe6f372 100644 --- a/tools/gate/playbooks/airskiff-reduce-site.yaml +++ b/tools/gate/playbooks/airskiff-reduce-site.yaml @@ -16,6 +16,7 @@ tasks: - name: Overwrite Armada manifest shell: | + set -ex git checkout v1.9 mv tools/gate/manifests/full-site.yaml \ type/skiff/manifests/full-site.yaml diff --git a/tools/gate/playbooks/run-integration-tests-docker.yaml b/tools/gate/playbooks/run-integration-tests-docker.yaml index 4a3b0664..abc0fc1f 100644 --- a/tools/gate/playbooks/run-integration-tests-docker.yaml +++ b/tools/gate/playbooks/run-integration-tests-docker.yaml @@ -23,7 +23,6 @@ - ensure-pip - disable-systemd-resolved - install-test-requirements - - build-images - deploy-keystone-dependencies - deploy-barbican - build-charts @@ -31,6 +30,7 @@ # deploy-keystone-dependencies which restarts Docker, causing # the first 2 containers that are deployed to fail. - deploy-postgresql + - build-images - generate-test-config - deploy-deckhand - run-integration-tests diff --git a/tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml b/tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml index 330ba26e..725fa4b0 100644 --- a/tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml +++ b/tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml @@ -17,30 +17,49 @@ set_fact: zuul_site_mirror_fqdn: "{{ lookup('env','zuul_site_mirror_fqdn') }}" -- name: Deploy Required Packages +- name: Checkout treasuremap v1.9 branch shell: | set -xe; - ./tools/deployment/common/install-packages.sh + git checkout v1.9 environment: zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" args: - chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" + chdir: "{{ zuul.project.src_dir }}/{{ zuul_treasuremap_relative_path | default('') }}" - name: Deploy Kubernetes shell: | set -xe; - ./tools/deployment/common/deploy-k8s.sh + ./tools/deployment/airskiff/developer/010-deploy-k8s.sh + environment: + zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_treasuremap_relative_path | default('') }}" + +- name: Make OSH charts + shell: | + set -xe + make all environment: zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" args: chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" -- name: Setup OS and Kubernetes Clients +- name: Make OSH_INFRA charts + shell: | + set -xe + make all + environment: + zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + + +- name: Setup docker-based openstack client shell: | set -xe; - ./tools/deployment/common/setup-client.sh + ./tools/deployment/airskiff/developer/020-setup-client.sh args: - chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" + chdir: "{{ zuul.project.src_dir }}/{{ zuul_treasuremap_relative_path | default('') }}" - name: Deploy Ingress shell: | diff --git a/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml b/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml index bc967f6b..759d1da3 100644 --- a/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml +++ b/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml @@ -15,18 +15,9 @@ - name: Install pip3 and gabbi shell: | set -xe; - apt-get install -y \ - python3-pip \ - python3-setuptools \ - python3-dev - pip3 install --upgrade pip - pip3 install -r test-requirements.txt - if test "{{ distro }}" = 'ubuntu_bionic' - then - pip3 install -r requirements-frozen-bionic.txt - else - pip3 install -r requirements-frozen.txt - fi + pip3 install \ + -r test-requirements.txt \ + -r requirements-frozen.txt args: chdir: "{{ zuul.project.src_dir }}" become: yes diff --git a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml index edc7d7ed..e1f2325c 100644 --- a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml +++ b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml @@ -55,13 +55,10 @@ # NOTE(felipemonteiro): We don't use a venv because they don't play nicely # with OpenStack-Helm, which is used to orchestrate various OpenStack # services in the integration script called below. - sudo -H -E pip3 install -e . --verbose --use-pep517 - if test "{{ distro }}" = 'ubuntu_bionic' - then - sudo -E -H pip3 install -r requirements-frozen-bionic.txt - else - sudo -E -H pip3 install -r requirements-frozen.txt - fi + sudo -E -H pip3 install -e . --verbose --use-pep517 + sudo -E -H pip3 install -r requirements-frozen.txt + echo "$(pwd)" | sudo tee /usr/local/lib/python3.8/dist-packages/deckhand.pth + sudo fdisk --list df -h sudo mkdir -p /opt/ext_vol diff --git a/tools/integration-tests.sh b/tools/integration-tests.sh index 0910ba9c..da53588b 100755 --- a/tools/integration-tests.sh +++ b/tools/integration-tests.sh @@ -102,6 +102,7 @@ function deploy_deckhand { cd ${CURRENT_DIR} + # TODO(fmontei): Use Keystone bootstrap override instead. interfaces=("admin" "public" "internal") deckhand_endpoint="http://127.0.0.1:9000" diff --git a/tox.ini b/tox.ini index 833d9811..e5d07524 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,7 @@ passenv = deps = -r{toxinidir}/requirements-frozen.txt -r{toxinidir}/test-requirements.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt commands = find . -type f -name "*.pyc" -delete rm -Rf .testrepository/times.dbm @@ -53,37 +54,12 @@ allowlist_externals= sh deps= -r{toxinidir}/requirements-direct.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt commands= rm -f requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt" -[testenv:freeze-bionic] -basepython=python3 -recreate = True -allowlist_externals= - rm - sh -deps= - -r{toxinidir}/requirements-direct.txt -commands= - rm -f requirements-frozen-bionic.txt - sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen-bionic.txt" - -[testenv:py36] -deps = - -r{toxinidir}/requirements-frozen-bionic.txt - -r{toxinidir}/test-requirements.txt -allowlist_externals = - stestr - find - rm -commands = - {[testenv]commands} - stestr run {posargs} - stestr slowest - - [testenv:py38] allowlist_externals = stestr @@ -103,35 +79,8 @@ allowlist_externals = find rm -[testenv:py36-postgresql] -deps = - -r{toxinidir}/requirements-frozen-bionic.txt - -r{toxinidir}/test-requirements.txt -commands = - {[testenv]commands} - bash {toxinidir}/tools/run_pifpaf.sh '{posargs}' -allowlist_externals = - bash - find - rm - -[testenv:py36-functional] -basepython=python3 -deps = - -r{toxinidir}/requirements-frozen-bionic.txt - -r{toxinidir}/test-requirements.txt -allowlist_externals = - find - stestr - pip3 - echo -setenv = VIRTUAL_ENV={envdir} -commands = - find . -type f -name "*.pyc" -delete - stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color {posargs} - [testenv:py38-functional] -basepython=python3 +basepython=python3.8 allowlist_externals = find stestr @@ -144,7 +93,7 @@ commands = [testenv:py38-functional-dev] -basepython=python3 +basepython=python3.8 # Minimalistic functional test job for running Deckhand functional tests # via uwsgi. Uses pifpaf for DB instantiation. Useful for developers. # Requires PostgreSQL be installed on host. @@ -159,33 +108,11 @@ commands = find . -type f -name "*.pyc" -delete pifpaf run postgresql -- {toxinidir}/tools/functional-tests.sh "{posargs}" -[testenv:py36-functional-dev] -basepython=python3 -# Minimalistic functional test job for running Deckhand functional tests -# via uwsgi. Uses pifpaf for DB instantiation. Useful for developers. -# Requires PostgreSQL be installed on host. -deps = - -r{toxinidir}/requirements-frozen-bionic.txt - -r{toxinidir}/test-requirements.txt -allowlist_externals = - find - stestr - pifpaf - pip3 -setenv = VIRTUAL_ENV={envdir} -commands = - pip3 install -e . --verbose --use-pep517 - find . -type f -name "*.pyc" -delete - pifpaf run postgresql -- {toxinidir}/tools/functional-tests.sh "{posargs}" - [testenv:cover] allowlist_externals = find stestr coverage -deps = - -r{toxinidir}/requirements-frozen.txt - -r{toxinidir}/test-requirements.txt setenv = {[testenv]setenv} PYTHON=coverage run --source deckhand --parallel-mode commands =