Airflow stable 2.6.2

This PS updates python modules and code to match Airflow 2.6.2:

- bionic py36 gates  were removed
- python code corrected to match new modules versions
- selection of python modules versions was perfrmed based on
  airflow-2.6.2 constraints
- airskiff deploy pipeline was aligned with latest in treasuremap v1.9
- postgresql image updated to 14.8

Change-Id: I65a1b86473ee3e988aae353b59fb5473d75851f9

.zuul.yaml

@@ -12,57 +12,42 @@
 
 - project:
     templates:
-      - openstack-cover-jobs
       - docs-on-readthedocs
       - openstack-python38-jobs
-      - openstack-python36-jobs
     vars:
       rtd_webhook_id: '38572'
       rtd_project_name: 'airship-deckhand'
     check:
       jobs:
-        - deckhand-tox-py36-postgresql
         - deckhand-tox-py38-postgresql
-        - deckhand-functional-uwsgi-py36
         - deckhand-functional-uwsgi-py38
-        - deckhand-functional-docker-py38-ubuntu_focal
+        - deckhand-functional-docker-py38
-        - deckhand-functional-docker-py36-ubuntu_bionic
-        - deckhand-integration-uwsgi-py36
         - deckhand-integration-uwsgi-py38
-        - deckhand-integration-docker-py38-ubuntu_focal
+        - deckhand-integration-docker-py38
-        - deckhand-integration-docker-py36-ubuntu_bionic
         - deckhand-chart-build-gate
         - deckhand-chart-build-latest-htk
-        - deckhand-docker-build-gate-ubuntu_focal
+        - deckhand-docker-build-gate
-        - deckhand-docker-build-gate-ubuntu_bionic
+        - deckhand-openstack-tox-pep8-focal
-        - openstack-tox-pep8
+        - deckhand-openstack-tox-cover-focal
         - deckhand-airskiff-deployment-focal
-        - deckhand-airskiff-deployment-bionic
+
     gate:
       jobs:
-        - deckhand-tox-py36-postgresql
         - deckhand-tox-py38-postgresql
-        - deckhand-functional-uwsgi-py36
         - deckhand-functional-uwsgi-py38
-        - deckhand-functional-docker-py38-ubuntu_focal
+        - deckhand-functional-docker-py38
-        - deckhand-functional-docker-py36-ubuntu_bionic
-        - deckhand-integration-uwsgi-py36
         - deckhand-integration-uwsgi-py38
-        - deckhand-integration-docker-py38-ubuntu_focal
+        - deckhand-integration-docker-py38
-        - deckhand-integration-docker-py36-ubuntu_bionic
         - deckhand-chart-build-gate
-        - deckhand-docker-build-gate-ubuntu_focal
+        - deckhand-docker-build-gate
-        - deckhand-docker-build-gate-ubuntu_bionic
+        - deckhand-openstack-tox-pep8-focal
-        - openstack-tox-pep8
+        - deckhand-openstack-tox-cover-focal
         - deckhand-airskiff-deployment-focal
-        - deckhand-airskiff-deployment-bionic
+
     post:
       jobs:
         - deckhand-upload-git-mirror
         - deckhand-docker-publish-ubuntu_focal
-        # - deckhand-docker-tag-ubuntu_focal
-        - deckhand-docker-publish-ubuntu_bionic
-        # - deckhand-docker-tag-ubuntu_bionic
 
 
 - nodeset:
@@ -89,6 +74,18 @@
       - name: primary
         label: ubuntu-focal
 
+- job:
+    name: deckhand-openstack-tox-pep8-focal
+    parent: openstack-tox-pep8
+    description: Runs pep8 job on focal
+    nodeset: deckhand-single-node-airskiff-focal
+
+- job:
+    name: deckhand-openstack-tox-cover-focal
+    parent: openstack-tox-cover
+    description: Runs cover job on focal
+    nodeset: deckhand-single-node-airskiff-focal
+
 - job:
     name: deckhand-tox-py38-postgresql
     parent: openstack-tox-py38
@@ -98,35 +95,6 @@
     vars:
       tox_envlist: py38-postgresql
 
-- job:
-    name: deckhand-tox-py36-postgresql
-    parent: openstack-tox-py38
-    nodeset: deckhand-single-node
-    pre-run:
-      - tools/gate/playbooks/install-postgresql.yaml
-    vars:
-      tox_envlist: py36-postgresql
-
-- job:
-    name: deckhand-functional-uwsgi-py36
-    voting: true
-    description: |
-      Run tox-based functional tests for the Airship Deckhand project using a
-      minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
-      for ephemeral PostgreSQL DB, under cPython version 3.6.
-    run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml
-    nodeset: deckhand-single-node
-    vars:
-      tox_envlist: py36-functional-dev
-      distro: ubuntu_bionic
-    irrelevant-files: &irrelevant-files
-      - ^.*\.rst$
-      - ^doc/.*$
-      - ^etc/.*$
-      - ^releasenotes/.*$
-      - ^setup.cfg$
-      - ^deckhand/tests/unit/.*$
-
 
 - job:
     name: deckhand-functional-uwsgi-py38
@@ -140,7 +108,13 @@
     vars:
       tox_envlist: py38-functional-dev
       distro: ubuntu_focal
-    irrelevant-files: *irrelevant-files
+    irrelevant-files: &irrelevant-files
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^setup.cfg$
+      - ^deckhand/tests/unit/.*$
 
 - job:
     name: deckhand-functional-docker-base
@@ -159,7 +133,7 @@
       - openstack/openstack-helm-infra
 
 - job:
-    name: deckhand-functional-docker-py38-ubuntu_focal
+    name: deckhand-functional-docker-py38
     description: |
       Run tox-based functional tests for the Airship Deckhand project under
       cPython version 3.8. Uses tox with the ``functional-py38`` environment.
@@ -172,36 +146,6 @@
       distro: ubuntu_focal
     irrelevant-files: *irrelevant-files
 
-- job:
-    name: deckhand-functional-docker-py36-ubuntu_bionic
-    description: |
-      Run tox-based functional tests for the Airship Deckhand project under
-      cPython version 3.6. Uses tox with the ``functional-py36`` environment.
-      Ubuntu (focal) image is built and used.
-    parent: deckhand-functional-docker-base
-    nodeset: deckhand-single-node
-    vars:
-      tox_envlist: py36-functional
-      disable_keystone: true
-      distro: ubuntu_bionic
-    irrelevant-files: *irrelevant-files
-
-
-
-- job:
-    name: deckhand-integration-uwsgi-py36
-    voting: true
-    description: |
-      Run tox-based integration tests for the Airship Deckhand project using a
-      minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
-      for ephemeral PostgreSQL DB, under cPython version 3.6.
-    timeout: 3600
-    run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml
-    nodeset: deckhand-single-node
-    irrelevant-files: *irrelevant-files
-    vars:
-      disable_keystone: true
-      distro: ubuntu_bionic
 
 - job:
     name: deckhand-integration-uwsgi-py38
@@ -210,7 +154,7 @@
       Run tox-based integration tests for the Airship Deckhand project using a
       minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
       for ephemeral PostgreSQL DB, under cPython version 3.8.
-    timeout: 3600
+    timeout: 9600
     run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml
     nodeset: deckhand-single-node-focal
     irrelevant-files: *irrelevant-files
@@ -248,6 +192,7 @@
     vars:
       zuul_osh_relative_path: ../../openstack/openstack-helm/
       zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra/
+      zuul_treasuremap_relative_path: ../../airship/treasuremap/
     pre-run:
       - tools/gate/playbooks/osh-infra-upgrade-host.yaml
       - tools/gate/playbooks/osh-infra-deploy-docker.yaml
@@ -257,13 +202,14 @@
     required-projects:
       - openstack/openstack-helm
       - openstack/openstack-helm-infra
+      - airship/treasuremap
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*$
       - ^releasenotes/.*$
 
 - job:
-    name: deckhand-integration-docker-py38-ubuntu_focal
+    name: deckhand-integration-docker-py38
     description: |
       Run tox-based integration tests for the Airship Deckhand project under
       cPython version 3.8. Builds ubuntu (focal) deckhand image.
@@ -273,17 +219,6 @@
       disable_keystone: false
       distro: ubuntu_focal
 
-- job:
-    name: deckhand-integration-docker-py36-ubuntu_bionic
-    description: |
-      Run tox-based integration tests for the Airship Deckhand project under
-      cPython version 3.6. Builds ubuntu (focal) deckhand image.
-    parent: deckhand-integration-docker-base
-    nodeset: openstack-helm-single-node
-    vars:
-      disable_keystone: false
-      distro: ubuntu_focal
-
 - job:
     name: deckhand-airskiff-deployment-focal
     voting: true
@@ -292,8 +227,8 @@
       Deploy Memcached using Airskiff and submitted Deckhand changes.
     timeout: 9600
     pre-run:
-      - tools/gate/playbooks/airskiff-reduce-site.yaml
       - tools/gate/playbooks/git-config.yaml
+      - tools/gate/playbooks/airskiff-reduce-site.yaml
     run: tools/gate/playbooks/airskiff-deploy.yaml
     post-run: tools/gate/playbooks/debug-report.yaml
     required-projects:
@@ -308,33 +243,9 @@
       - ^doc/.*$
       - ^releasenotes/.*$
 
-- job:
-    name: deckhand-airskiff-deployment-bionic
-    voting: true
-    nodeset: deckhand-single-node-airskiff-focal
-    description: |
-      Deploy Memcached using Airskiff and submitted Deckhand changes.
-    timeout: 9600
-    pre-run:
-      - tools/gate/playbooks/airskiff-reduce-site.yaml
-      - tools/gate/playbooks/git-config.yaml
-    run: tools/gate/playbooks/airskiff-deploy.yaml
-    post-run: tools/gate/playbooks/debug-report.yaml
-    required-projects:
-      - airship/treasuremap
-    vars:
-      CLONE_DECKHAND: false
-      OSH_INFRA_COMMIT: 443ff3e3e340c94c5cbb214d1e2a8b2a3937541d
-      DECKHAND_IMAGE_DISTRO: ubuntu_bionic
-      distro: ubuntu_focal
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^doc/.*$
-      - ^releasenotes/.*$
-
 
 - job:
-    name: deckhand-docker-build-gate-ubuntu_focal
+    name: deckhand-docker-build-gate
     timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: deckhand-single-node-focal
@@ -352,19 +263,6 @@
         dynamic:
           patch_set: true
 
-- job:
-    name: deckhand-docker-build-gate-ubuntu_bionic
-    timeout: 3600
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: deckhand-single-node
-    irrelevant-files: *non-code-files-template
-    vars:
-      publish: false
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          patch_set: true
-
 - job:
     name: deckhand-docker-publish-ubuntu_focal
     description: |
@@ -386,61 +284,23 @@
           commit: true
         static:
           - latest
+          - airflow_2.6.2
 
 - job:
-    name: deckhand-docker-publish-ubuntu_bionic
+    name: deckhand-docker-tag-ubuntu_focal
     description: |
-      Runs on every merge, unless files in a dictionary below are changed.
+      Runs on every merge when files in a dictionalry below are changed, and
-      Builds and publishes container ubuntu images on quay.io with a set of tags
+      adds git commit id tag onto the ubuntu container image published on quay.io,
-      listed in vars section. Waits in Zuul queue for a node (VM) assignment.
+      which has `latest` tag set. Does not wait in queue for a node (VM)
+      assignment, runs almost immediately.
     timeout: 3600
-    run: tools/gate/playbooks/docker-image-build.yaml
+    run: tools/gate/playbooks/docker-image-tag.yaml
-    nodeset: deckhand-single-node
+    nodeset:
+      nodes: []
     secrets:
       - airship_deckhand_quay_creds
-    irrelevant-files: *non-code-files-template
     vars:
-      publish: true
+      distro: ubuntu_focal
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          branch: true
-          commit: true
-        static:
-          - latest
-
-# - job:
-#     name: deckhand-docker-tag-ubuntu_focal
-#     description: |
-#       Runs on every merge when files in a dictionalry below are changed, and
-#       adds git commit id tag onto the ubuntu container image published on quay.io,
-#       which has `latest` tag set. Does not wait in queue for a node (VM)
-#       assignment, runs almost immediately.
-#     timeout: 3600
-#     run: tools/gate/playbooks/docker-image-tag.yaml
-#     nodeset:
-#       nodes: []
-#     secrets:
-#       - airship_deckhand_quay_creds
-#     vars:
-#       distro: ubuntu_focal
-
-# - job:
-#     name: deckhand-docker-tag-ubuntu_bionic
-#     description: |
-#       Runs on every merge when files in a dictionalry below are changed, and
-#       adds git commit id tag onto the ubuntu container image published on quay.io,
-#       which has `latest` tag set. Does not wait in queue for a node (VM)
-#       assignment, runs almost immediately.
-#     timeout: 3600
-#     run: tools/gate/playbooks/docker-image-tag.yaml
-#     nodeset:
-#       nodes: []
-#     secrets:
-#       - airship_deckhand_quay_creds
-#     vars:
-#       distro: ubuntu_bionic
-
 
 
 - secret:

ChangeLog

@@ -1,6 +1,9 @@
 CHANGES
 =======
 
+* Airflow stable 2.6.2
+* Restored ubuntu\_bionic image build
+* [focal] Fix requests.body attribute deprecation
 * Update airskiff deployment gate
 * Deckhand updates
 * Removing egg-info folder

deckhand/conf/config.py

@@ -43,7 +43,7 @@ barbican_opts = [
                 help="Whether to enable Barbican secret caching. Useful "
                      "for testing to avoid cross-test caching conflicts."),
     cfg.StrOpt(
-        'cache_timeout', default=3600,
+        'cache_timeout', default='3600',
         help="How long (in seconds) Barbican secret reference/payload lookup "
              "results should remain cached in memory.")
 ]
@@ -63,7 +63,7 @@ engine_opts = [
     cfg.BoolOpt('enable_cache', default=True,
                 help="Whether to enable the document rendering caching. Useful"
                      " for testing to avoid cross-test caching conflicts."),
-    cfg.IntOpt('cache_timeout', default=3600,
+    cfg.IntOpt('cache_timeout', default='3600',
                help="How long (in seconds) document rendering results should "
                     "remain cached in memory."),
 ]
@@ -76,7 +76,7 @@ jsonpath_group = cfg.OptGroup(
 
 
 jsonpath_opts = [
-    cfg.IntOpt('cache_timeout', default=3600,
+    cfg.IntOpt('cache_timeout', default='3600',
                help="How long (in seconds) JSONPath lookup results should "
                     "remain cached in memory.")
 ]

deckhand/tests/common/test_gabbi.py

@@ -90,7 +90,7 @@ class MultidocJsonpaths(jsonhandler.JSONHandler):
         # NOTE: The simple approach to handling dictionary versus list response
         # bodies is to always parse the response body as a list and index into
         # the first element using [0] throughout the tests.
-        return list(yaml.load_all(string))
+        return list(yaml.safe_load_all(string))
 
 
 def load_tests(loader, tests, pattern):

entrypoint.sh

@@ -38,13 +38,13 @@ echo "Command: $1 with arguments $@"
 # Start deckhand application
 if [ "$1" = 'server' ]; then
     exec uwsgi \
-        -b 32768 \
+        --buffer-size 32768 \
         --callable deckhand_callable \
         --die-on-term \
         --enable-threads \
         --http :${PORT} \
         --http-timeout ${DECKHAND_API_TIMEOUT} \
-        -L \
+        --disable-logging \
         --lazy-apps \
         --master \
         --pyargv "--config-file ${DECKHAND_CONFIG_DIR}/deckhand.conf" \
@@ -53,7 +53,7 @@ if [ "$1" = 'server' ]; then
         --strict \
         --vacuum \
         --need-app \
-        -w deckhand.cmd
+        --module deckhand.cmd
 elif [ "$1" = 'alembic' ]; then
     exec alembic ${@:2}
 else

etc/deckhand/deckhand.conf.sample

@@ -114,7 +114,7 @@
 
 # Format string to use for log messages with context. Used by
 # oslo_log.formatters.ContextFormatter (string value)
-#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
+#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s
 
 # Format string to use for log messages when context is undefined. Used by
 # oslo_log.formatters.ContextFormatter (string value)
@@ -131,7 +131,7 @@
 # Defines the format string for %(user_identity)s that is used in
 # logging_context_format_string. Used by oslo_log.formatters.ContextFormatter
 # (string value)
-#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s
+#logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s
 
 # List of package logging levels in logger=LEVEL pairs. This option is ignored
 # if log_config_append is set. (list value)
@@ -242,7 +242,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>
 
-# ID of the trust to use as a trustee use (string value)
+# Trust ID (string value)
 #trust_id = <None>
 
 # User ID (string value)
@@ -508,7 +508,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>
 
-# ID of the trust to use as a trustee use (string value)
+# Trust ID (string value)
 #trust_id = <None>
 
 # Optional domain ID to use with v3 and v2 parameters. It will be used for both
@@ -718,6 +718,9 @@
 # middleware should parse the headers or not. (boolean value)
 #enable_proxy_headers_parsing = false
 
+# HTTP basic auth password file. (string value)
+#http_basic_auth_user_file = /etc/htpasswd
+
 
 [oslo_policy]
 

requirements-direct.txt

@@ -1,77 +1,28 @@
-alembic<=1.4.3
+barbican==14.0.2
-amqp==2.6.1
+Beaker
-Beaker<=1.12.0
+deepdiff
-chardet==3.0.4
-charset_normalizer >=2.0.0, < 4.0.0
-ConfigParser
-coverage
-cryptography==3.4.8
-deepdiff<=5.8.1
-docutils<=0.17
 falcon
-hacking
+fixtures
-importlib-metadata~=2.0
-Jinja2<=3.0.3
 jsonpath_ng
-jsonpath-rw-ext>=1.0.0
+jsonpickle
-jsonpickle==1.4.1
+jsonschema
-jsonschema<=3.2.0
+keystoneauth1==4.3.1
-keystoneauth1<=5.1.1
-keystonemiddleware
-kombu<=4.6.11
-MarkupSafe<2.1.0, >=0.9.2
 networkx
-nose
+oslo.config==8.6.0
-oslo.cache<=2.10.1
+oslo.context==4.0.0
-oslo.concurrency
+oslo.log==4.8.0
-oslo.config<=8.7.1
+oslo.policy==3.10.1
-oslo.context<=4.1.0
+oslo.serialization==4.1.0
-oslo.db<=10.0.0
+oslo.utils==4.12.3
-oslo.log<=4.6.0
+oslo.db==11.2.0
-oslo.messaging<=12.13.0
-oslo.middleware<=4.4.0
-oslo.policy<=3.10.1
-oslo.serialization<=4.2.0
-oslo.utils<=4.12.3
-packaging==21.3
-Paste<=3.5.0
-PasteDeploy
 PasteScript
-pbr<=5.5.1
+pylibyaml
-psycopg2-binary
+python-barbicanclient
-pycadf<=3.1.1
-pycodestyle<=2.6.0
-pyflakes<=2.2.0
-Pygments<=2.14.0
-pylibyaml==0.1.0
-pymongo
-pyparsing<=2.4.7
-pyproject-api
-python-barbicanclient<=5.2.0
-python-dateutil==2.8.1
-python-keystoneclient
 python-memcached
-python-subunit<=1.4.0
+PyYAML
-PyYAML<=5.4.1
-reno
-requests==2.23.0
-Routes
-setuptools<=45.2.0
 six
-Sphinx==3.3.1
+testtools
-sphinx-rtd-theme==0.5.0
-SQLAlchemy<=1.3.20
-stevedore
-testrepository
-testresources
-testscenarios
-testtools<=2.5.0
 tiddlyweb
-typing-extensions==3.7.4.3
-urllib3 >= 1.21.1, <= 1.25.11
-uWSGI==2.0.21
-virtualenv
 Werkzeug
-wheel
+psycopg2-binary
-wsgi-intercept>=1.2.2
+uwsgi
-yq==3.2.1

requirements-frozen-bionic.txt

@@ -1,156 +0,0 @@
-alabaster==0.7.13
-alembic==1.4.3
-amqp==2.6.1
-argcomplete==3.0.8
-attrs==22.2.0
-autopage==0.5.1
-Babel==2.11.0
-bcrypt==4.0.1
-Beaker==1.12.0
-cachetools==4.2.4
-certifi==2023.5.7
-cffi==1.15.1
-chardet==3.0.4
-charset-normalizer==3.0.1
-cliff==3.10.1
-cmd2==2.4.3
-configparser==5.2.0
-coverage==6.2
-cryptography==3.4.8
-dataclasses==0.8
-debtcollector==2.5.0
-decorator==4.4.2
-deepdiff==5.7.0
-distlib==0.3.6
-dnspython==2.2.1
-docutils==0.17
-dogpile.cache==1.2.0
-dulwich==0.20.50
-eventlet==0.33.3
-extras==1.0.0
-falcon==3.1.1
-fasteners==0.18
-filelock==3.4.1
-fixtures==3.0.0
-flake8==3.8.4
-futurist==2.4.1
-greenlet==2.0.2
-hacking==4.1.0
-html5lib==0.9999999
-httpexceptor==1.4.0
-idna==2.10
-imagesize==1.4.1
-importlib-metadata==2.1.3
-importlib-resources==5.4.0
-iso8601==1.1.0
-Jinja2==3.0.3
-jsonpath-ng==1.5.3
-jsonpath-rw==1.4.0
-jsonpath-rw-ext==1.2.2
-jsonpickle==1.4.1
-jsonschema==3.2.0
-keystoneauth1==5.1.1
-keystonemiddleware==9.5.0
-kombu==4.6.11
-Mako==1.1.6
-MarkupSafe==2.0.1
-mccabe==0.6.1
-msgpack==1.0.5
-netaddr==0.8.0
-netifaces==0.11.0
-networkx==2.5.1
-nose==1.3.7
-ordered-set==4.0.2
-os-service-types==1.7.0
-oslo.cache==2.10.1
-oslo.concurrency==4.5.1
-oslo.config==8.7.1
-oslo.context==4.1.0
-oslo.db==10.0.0
-oslo.i18n==5.1.0
-oslo.log==4.6.0
-oslo.messaging==12.13.0
-oslo.metrics==0.4.0
-oslo.middleware==4.4.0
-oslo.policy==3.10.1
-oslo.serialization==4.2.0
-oslo.service==2.8.0
-oslo.utils==4.12.3
-packaging==21.3
-Paste==3.5.0
-PasteDeploy==2.1.1
-PasteScript==3.3.0
-pbr==5.5.1
-pip==21.3.1
-platformdirs==2.4.0
-ply==3.11
-prettytable==2.5.0
-prometheus-client==0.16.0
-psycopg2-binary==2.9.6
-pycadf==3.1.1
-pycodestyle==2.6.0
-pycparser==2.21
-pyflakes==2.2.0
-Pygments==2.14.0
-pylibyaml==0.1.0
-pymongo==4.1.1
-pyparsing==2.4.7
-pyperclip==1.8.2
-pyproject-api==0.1.0
-pyrsistent==0.18.0
-python-barbicanclient==5.2.0
-python-dateutil==2.8.1
-python-editor==1.0.4
-python-keystoneclient==4.5.0
-python-memcached==1.59
-python-mimeparse==1.6.0
-python-subunit==1.4.0
-pytz==2023.3
-PyYAML==5.4.1
-reno==4.0.0
-repoze.lru==0.7
-requests==2.23.0
-resolver==0.2.1
-rfc3986==1.5.0
-Routes==2.5.1
-selector==0.10.1
-setuptools==45.2.0
-simplejson==3.19.1
-six==1.16.0
-snowballstemmer==2.2.0
-Sphinx==3.3.1
-sphinx-rtd-theme==0.5.0
-sphinxcontrib-applehelp==1.0.2
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.0
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-SQLAlchemy==1.3.20
-sqlalchemy-migrate==0.13.0
-sqlparse==0.4.4
-statsd==4.0.1
-stevedore==3.5.2
-Tempita==0.5.2
-testrepository==0.0.20
-testresources==2.0.1
-testscenarios==0.5.0
-testtools==2.5.0
-tiddlyweb==2.4.3
-tomli==1.2.3
-tomlkit==0.11.6
-typing-extensions==3.7.4.3
-urllib3==1.25.11
-uWSGI==2.0.21
-vine==1.3.0
-virtualenv==20.16.2
-wcwidth==0.2.6
-WebOb==1.8.7
-Werkzeug==2.0.3
-wheel==0.37.1
-wrapt==1.15.0
-wsgi-intercept==1.11.0
-xmltodict==0.13.0
-yappi==1.4.0
-yq==3.2.1
-zipp==3.6.0

requirements-frozen.txt

@@ -1,154 +1,124 @@
-alabaster==0.7.13
+alembic==1.11.1
-alembic==1.4.3
+amqp==5.1.1
-amqp==2.6.1
-argcomplete==3.0.8
 attrs==23.1.0
 autopage==0.5.1
-Babel==2.12.1
+backports.zoneinfo==0.2.1
+barbican==14.0.2
 bcrypt==4.0.1
-Beaker==1.12.0
+Beaker==1.12.1
-cachetools==5.3.0
+cachetools==5.3.1
-certifi==2022.12.7
+castellan==4.1.0
+certifi==2023.5.7
 cffi==1.15.1
-chardet==3.0.4
 charset-normalizer==3.1.0
-cliff==3.10.1
+cliff==4.3.0
 cmd2==2.4.3
-configparser==5.3.0
+cryptography==40.0.2
-coverage==7.2.3
-cryptography==3.4.8
 debtcollector==2.5.0
 decorator==5.1.1
-deepdiff==5.8.1
+deepdiff==6.3.1
-distlib==0.3.6
 dnspython==2.3.0
-docutils==0.17
+dogpile.cache==1.2.2
-dogpile.cache==1.1.8
-dulwich==0.21.3
 eventlet==0.33.3
-extras==1.0.0
 falcon==3.1.1
 fasteners==0.18
-filelock==3.12.0
+fixtures==4.1.0
-fixtures==3.0.0
-flake8==3.8.4
 futurist==2.4.1
 greenlet==2.0.2
-hacking==4.1.0
 html5lib==0.9999999
 httpexceptor==1.4.0
-idna==2.10
+idna==3.4
-imagesize==1.4.1
+importlib-metadata==4.13.0
-importlib-metadata==2.1.3
+importlib-resources==5.12.0
-iso8601==1.1.0
+iso8601==2.0.0
-Jinja2==3.0.3
+Jinja2==3.1.2
 jsonpath-ng==1.5.3
-jsonpath-rw==1.4.0
+jsonpickle==3.0.1
-jsonpath-rw-ext==1.2.2
+jsonschema==4.17.3
-jsonpickle==1.4.1
+keystoneauth1==4.3.1
-jsonschema==3.2.0
+keystonemiddleware==10.4.0
-keystoneauth1==5.1.1
+kombu==5.3.0
-keystonemiddleware==10.2.0
+ldap3==2.9.1
-kombu==4.6.11
+logutils==0.3.5
 Mako==1.2.4
-MarkupSafe==2.0.1
+MarkupSafe==2.1.3
-mccabe==0.6.1
 msgpack==1.0.5
 netaddr==0.8.0
 netifaces==0.11.0
 networkx==3.1
-nose==1.3.7
 ordered-set==4.1.0
 os-service-types==1.7.0
-oslo.cache==2.10.1
+oslo.cache==3.4.0
 oslo.concurrency==5.1.1
-oslo.config==8.7.1
+oslo.config==8.6.0
-oslo.context==4.1.0
+oslo.context==4.0.0
-oslo.db==10.0.0
+oslo.db==11.2.0
 oslo.i18n==6.0.0
-oslo.log==4.6.0
+oslo.log==4.8.0
-oslo.messaging==12.13.0
+oslo.messaging==14.3.1
 oslo.metrics==0.6.0
-oslo.middleware==4.4.0
+oslo.middleware==5.1.1
 oslo.policy==3.10.1
-oslo.serialization==4.2.0
+oslo.serialization==4.1.0
 oslo.service==3.1.1
+oslo.upgradecheck==2.1.1
 oslo.utils==4.12.3
+oslo.versionedobjects==3.1.0
 packaging==21.3
-Paste==3.5.0
+Paste==3.5.3
 PasteDeploy==3.0.1
 PasteScript==3.3.0
-pbr==5.5.1
+pbr==5.11.1
-pip==23.0.1
+pecan==1.5.1
-platformdirs==3.2.0
+pip==23.1.2
+pkgutil_resolve_name==1.3.10
 ply==3.11
-prettytable==3.7.0
+prettytable==3.8.0
-prometheus-client==0.16.0
+prometheus-client==0.17.0
 psycopg2-binary==2.9.6
+pyasn1==0.4.8
 pycadf==3.1.1
-pycodestyle==2.6.0
 pycparser==2.21
-pyflakes==2.2.0
-Pygments==2.14.0
 pylibyaml==0.1.0
-pymongo==4.3.3
+pyOpenSSL==23.2.0
-pyparsing==2.4.7
+pyparsing==3.0.9
 pyperclip==1.8.2
-pyproject_api==1.5.0
 pyrsistent==0.19.3
-python-barbicanclient==5.2.0
+python-barbicanclient==5.4.0
-python-dateutil==2.8.1
+python-dateutil==2.8.2
-python-editor==1.0.4
 python-keystoneclient==5.1.0
 python-memcached==1.59
 python-mimeparse==1.6.0
-python-subunit==1.4.0
 pytz==2023.3
-PyYAML==5.4.1
+PyYAML==6.0
-reno==4.0.0
 repoze.lru==0.7
-requests==2.23.0
+requests==2.31.0
 resolver==0.2.1
-rfc3986==2.0.0
+rfc3986==1.5.0
 Routes==2.5.1
 selector==0.10.1
-setuptools==45.2.0
+setuptools==67.7.2
 simplejson==3.19.1
 six==1.16.0
-snowballstemmer==2.2.0
+SQLAlchemy==1.4.48
-Sphinx==3.3.1
-sphinx-rtd-theme==0.5.0
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.1
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-SQLAlchemy==1.3.20
 sqlalchemy-migrate==0.13.0
 sqlparse==0.4.4
 statsd==4.0.1
-stevedore==5.0.0
+stevedore==5.1.0
 Tempita==0.5.2
-testrepository==0.0.20
 testresources==2.0.1
 testscenarios==0.5.0
-testtools==2.5.0
+testtools==2.6.0
 tiddlyweb==2.4.3
-tomli==2.0.1
+typing_extensions==4.6.3
-tomlkit==0.11.7
+tzdata==2023.3
-typing-extensions==3.7.4.3
+urllib3==1.26.16
-urllib3==1.25.11
+uWSGI==2.0.22
-uWSGI==2.0.21
+vine==5.0.0
-vine==1.3.0
-virtualenv==20.22.0
 wcwidth==0.2.6
 WebOb==1.8.7
-Werkzeug==2.1.2
+Werkzeug==2.2.3
 wheel==0.40.0
 wrapt==1.15.0
-wsgi-intercept==1.11.0
-xmltodict==0.13.0
 yappi==1.4.0
-yq==3.2.1
 zipp==3.15.0

test-requirements.txt

@@ -1,29 +1,29 @@
 
 
-amqp
+# amqp
 bandit==1.6.0
-chardet==3.0.4
+# chardet==3.0.4
-fixtures==3.0.0
+# fixtures==3.0.0
 flake8==3.8.4
 gabbi==1.35.1
-jsonpickle==1.4.1
+# jsonpickle==1.4.1
+# openstacksdk==0.36.5
 openstacksdk==0.36.5
-openstacksdk==0.36.5
+# os-testr==2.0.1
-os-testr==2.0.1
 os-testr==2.0.1
 oslotest==4.5.0
-packaging==21.3
+# packaging==21.3
 pifpaf==3.1.5
-pylibyaml==0.1.0
+# pylibyaml==0.1.0
-pyparsing==2.4.7
+# pyparsing==2.4.7
 pytest >= 3.0
-pytest-cov==4.0.0
+pytest-cov
 python-subunit==1.4.0
-requests==2.23.0
+# requests==2.23.0
-six==1.16.0
+# six==1.16.0
 stestr==3.2.0
 testrepository==0.0.20
-testtools==2.5.0
+# testtools==2.5.0
 tox>=3.28.0, <4.0.0
-urllib3==1.25.11
+# urllib3==1.25.11
-yq==3.2.1
+# yq==3.2.1

tools/gate/playbooks/airskiff-deploy.yaml

@@ -17,6 +17,8 @@
     - clear-firewall
     - bindep
     - ensure-docker
+    - ensure-python
+    - ensure-pip
     - disable-systemd-resolved
     - install-test-requirements
 
@@ -31,6 +33,7 @@
 
     - name: Clone Required Repositories
       shell: |
+        set -ex
         export CLONE_DECKHAND={{ CLONE_DECKHAND }}
         export OSH_INFRA_COMMIT={{ OSH_INFRA_COMMIT }}
         ./tools/deployment/airskiff/developer/000-clone-dependencies.sh
@@ -60,6 +63,14 @@
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
 
+    - name: Setup openstack client
+      shell: |
+        set -ex
+        ./tools/deployment/airskiff/developer/020-setup-client.sh
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
     - name: Build Deckhand with submitted changes
       shell: |
         set -ex
@@ -76,18 +87,10 @@
         export DISTRO={{ DECKHAND_IMAGE_DISTRO }}
         docker rm registry --force || true
         docker run -d -p 5000:5000 --restart=always --name registry registry:2
-        if test "${DISTRO}" = 'ubuntu_bionic'
+        docker tag quay.io/airshipit/deckhand:latest-ubuntu_focal localhost:5000/deckhand:latest-ubuntu_focal
-        then
+        docker push localhost:5000/deckhand:latest-ubuntu_focal
-          docker tag quay.io/airshipit/deckhand:latest-ubuntu_bionic localhost:5000/deckhand:latest-ubuntu_bionic
+        sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml
-          docker push localhost:5000/deckhand:latest-ubuntu_bionic
+        sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_focal#g" ./global/software/config/versions.yaml
-          sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_bionic#g" ./site/airskiff/software/config/versions.yaml
-          sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_bionic#g" ./global/software/config/versions.yaml
-        else
-          docker tag quay.io/airshipit/deckhand:latest-ubuntu_focal localhost:5000/deckhand:latest-ubuntu_focal
-          docker push localhost:5000/deckhand:latest-ubuntu_focal
-          sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml
-          sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_focal#g" ./global/software/config/versions.yaml
-        fi
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
       become: yes
@@ -116,6 +119,7 @@
       shell: |
         mkdir -p ~/.kube
         cp -rp /home/zuul/.kube/config ~/.kube/config
+        pip3 install packaging
         ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
@@ -130,6 +134,7 @@
 
     - name: Wait for deployment completion
       shell: |
+        set -x
         ./tools/gate/wait-for-shipyard.sh
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"

tools/gate/playbooks/airskiff-reduce-site.yaml

@@ -16,6 +16,7 @@
   tasks:
     - name: Overwrite Armada manifest
       shell: |
+        set -ex
         git checkout v1.9
         mv tools/gate/manifests/full-site.yaml \
             type/skiff/manifests/full-site.yaml

tools/gate/playbooks/run-integration-tests-docker.yaml

@@ -23,7 +23,6 @@
     - ensure-pip
     - disable-systemd-resolved
     - install-test-requirements
-    - build-images
     - deploy-keystone-dependencies
     - deploy-barbican
     - build-charts
@@ -31,6 +30,7 @@
     # deploy-keystone-dependencies which restarts Docker, causing
     # the first 2 containers that are deployed to fail.
     - deploy-postgresql
+    - build-images
     - generate-test-config
     - deploy-deckhand
     - run-integration-tests

tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml

@@ -17,30 +17,49 @@
   set_fact:
     zuul_site_mirror_fqdn: "{{ lookup('env','zuul_site_mirror_fqdn') }}"
 
-- name: Deploy Required Packages
+- name: Checkout treasuremap v1.9 branch
   shell: |
     set -xe;
-    ./tools/deployment/common/install-packages.sh
+    git checkout v1.9
   environment:
     zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
   args:
-    chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}"
+    chdir: "{{ zuul.project.src_dir }}/{{ zuul_treasuremap_relative_path | default('') }}"
 
 - name: Deploy Kubernetes
   shell: |
     set -xe;
-    ./tools/deployment/common/deploy-k8s.sh
+    ./tools/deployment/airskiff/developer/010-deploy-k8s.sh
+  environment:
+    zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+  args:
+    chdir: "{{ zuul.project.src_dir }}/{{ zuul_treasuremap_relative_path | default('') }}"
+
+- name: Make OSH charts
+  shell: |
+    set -xe
+    make all
   environment:
     zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
   args:
     chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}"
 
-- name: Setup OS and Kubernetes Clients
+- name: Make OSH_INFRA charts
+  shell: |
+    set -xe
+    make all
+  environment:
+    zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+  args:
+    chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+
+
+- name: Setup docker-based openstack client
   shell: |
     set -xe;
-    ./tools/deployment/common/setup-client.sh
+    ./tools/deployment/airskiff/developer/020-setup-client.sh
   args:
-    chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}"
+    chdir: "{{ zuul.project.src_dir }}/{{ zuul_treasuremap_relative_path | default('') }}"
 
 - name: Deploy Ingress
   shell: |

tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml

@@ -15,18 +15,9 @@
 - name: Install pip3 and gabbi
   shell: |
     set -xe;
-    apt-get install -y \
+    pip3 install \
-      python3-pip \
+      -r test-requirements.txt \
-      python3-setuptools \
+      -r requirements-frozen.txt
-      python3-dev
-    pip3 install --upgrade pip
-    pip3 install -r test-requirements.txt
-    if test "{{ distro }}" = 'ubuntu_bionic'
-    then
-      pip3 install -r requirements-frozen-bionic.txt
-    else
-      pip3 install -r requirements-frozen.txt
-    fi
   args:
     chdir: "{{ zuul.project.src_dir }}"
   become: yes

tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml

@@ -55,13 +55,10 @@
     # NOTE(felipemonteiro): We don't use a venv because they don't play nicely
     # with OpenStack-Helm, which is used to orchestrate various OpenStack
     # services in the integration script called below.
-    sudo -H -E pip3 install -e . --verbose --use-pep517
+    sudo -E -H pip3 install -e . --verbose --use-pep517
-    if test "{{ distro }}" = 'ubuntu_bionic'
+    sudo -E -H pip3 install -r requirements-frozen.txt
-    then
+    echo "$(pwd)" | sudo tee /usr/local/lib/python3.8/dist-packages/deckhand.pth
-      sudo -E -H pip3 install -r requirements-frozen-bionic.txt
+
-    else
-      sudo -E -H pip3 install -r requirements-frozen.txt
-    fi
     sudo fdisk --list
     df -h
     sudo mkdir -p /opt/ext_vol

tools/integration-tests.sh

@@ -102,6 +102,7 @@ function deploy_deckhand {
 
     cd ${CURRENT_DIR}
 
+
     # TODO(fmontei): Use Keystone bootstrap override instead.
     interfaces=("admin" "public" "internal")
     deckhand_endpoint="http://127.0.0.1:9000"

tox.ini

@@ -34,6 +34,7 @@ passenv =
 deps =
   -r{toxinidir}/requirements-frozen.txt
   -r{toxinidir}/test-requirements.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
 commands =
   find . -type f -name "*.pyc" -delete
   rm -Rf .testrepository/times.dbm
@@ -53,37 +54,12 @@ allowlist_externals=
   sh
 deps=
   -r{toxinidir}/requirements-direct.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
 commands=
   rm -f requirements-frozen.txt
   sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt"
 
 
-[testenv:freeze-bionic]
-basepython=python3
-recreate = True
-allowlist_externals=
-  rm
-  sh
-deps=
-  -r{toxinidir}/requirements-direct.txt
-commands=
-  rm -f requirements-frozen-bionic.txt
-  sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen-bionic.txt"
-
-[testenv:py36]
-deps =
-  -r{toxinidir}/requirements-frozen-bionic.txt
-  -r{toxinidir}/test-requirements.txt
-allowlist_externals =
-  stestr
-  find
-  rm
-commands =
-  {[testenv]commands}
-  stestr run {posargs}
-  stestr slowest
-
-
 [testenv:py38]
 allowlist_externals =
   stestr
@@ -103,35 +79,8 @@ allowlist_externals =
   find
   rm
 
-[testenv:py36-postgresql]
-deps =
-  -r{toxinidir}/requirements-frozen-bionic.txt
-  -r{toxinidir}/test-requirements.txt
-commands =
-    {[testenv]commands}
-    bash {toxinidir}/tools/run_pifpaf.sh '{posargs}'
-allowlist_externals =
-  bash
-  find
-  rm
-
-[testenv:py36-functional]
-basepython=python3
-deps =
-  -r{toxinidir}/requirements-frozen-bionic.txt
-  -r{toxinidir}/test-requirements.txt
-allowlist_externals =
-  find
-  stestr
-  pip3
-  echo
-setenv = VIRTUAL_ENV={envdir}
-commands =
-  find . -type f -name "*.pyc" -delete
-  stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color {posargs}
-
 [testenv:py38-functional]
-basepython=python3
+basepython=python3.8
 allowlist_externals =
   find
   stestr
@@ -144,7 +93,7 @@ commands =
 
 
 [testenv:py38-functional-dev]
-basepython=python3
+basepython=python3.8
 # Minimalistic functional test job for running Deckhand functional tests
 # via uwsgi. Uses pifpaf for DB instantiation. Useful for developers.
 # Requires PostgreSQL be installed on host.
@@ -159,33 +108,11 @@ commands =
   find . -type f -name "*.pyc" -delete
   pifpaf run postgresql -- {toxinidir}/tools/functional-tests.sh "{posargs}"
 
-[testenv:py36-functional-dev]
-basepython=python3
-# Minimalistic functional test job for running Deckhand functional tests
-# via uwsgi. Uses pifpaf for DB instantiation. Useful for developers.
-# Requires PostgreSQL be installed on host.
-deps =
-  -r{toxinidir}/requirements-frozen-bionic.txt
-  -r{toxinidir}/test-requirements.txt
-allowlist_externals =
-  find
-  stestr
-  pifpaf
-  pip3
-setenv = VIRTUAL_ENV={envdir}
-commands =
-  pip3 install -e . --verbose --use-pep517
-  find . -type f -name "*.pyc" -delete
-  pifpaf run postgresql -- {toxinidir}/tools/functional-tests.sh "{posargs}"
-
 [testenv:cover]
 allowlist_externals =
   find
   stestr
   coverage
-deps =
-  -r{toxinidir}/requirements-frozen.txt
-  -r{toxinidir}/test-requirements.txt
 setenv = {[testenv]setenv}
          PYTHON=coverage run --source deckhand --parallel-mode
 commands =