From 70aa35a396d5f76753616f5289228f9c2b0e7ec7 Mon Sep 17 00:00:00 2001 From: "Wahlstedt, Walter (ww229g)" Date: Thu, 5 Jan 2023 12:35:38 -0500 Subject: [PATCH] update to focal and python 3.8 update dockerfile for python deckhand install add deckhand version to chart 1.0 add chart version 0.2.0 update all packages to latest in requirements.txt update zuul jobs for focal and python 3.8 remove zuul job functional-uwsgi-py38 in favor of functional-docker-py38 update tox config typecast to string in re.sub() function add stestr to test-requirements.txt add SQLAlchemy jsonpickle sphinx-rtd-theme stestr to requirements.txt deprecated function: BarbicanException -> BarbicanClientException fix mock import using unittest fix import collections to collections.abc fix for collections modules for older than python 3.10 versions. deprecated function: json -> to_json deprecated function: werkzeug.contrib.profiler -> werkzeug.middleware.profiler deprecated function: falcon.AIP -> falcon.App deprecation warning: switch from resp.body to resp.text rename fixtures to dh_fixtures because there is an imported module fixtures switch from stream.read to bounded_stream.read deprecated function: falcon process_response needed additional parameter deprecated function: falcon default_exception_handler changed parameter order move from MagicMock object to falcon test generated object to fix incompatability with upgraded Falcon module. Adjust gabbi tests to fix incompatability with upgraded DeepDiff module update Makefile to execute ubuntu_focal update HTK (helmtoolkit) unpin barbican to pass integration tests Use helm 3 in chart build. `helm serve` is removed in helm 3 so this moves to using local `file://` dependencies [0] instead. Change-Id: I180416f480edea1b8968d80c993b3e1fcc95c08d --- .dockerignore | 123 +++++++++- .gitignore | 1 + .zuul.yaml | 215 ++++++------------ Makefile | 8 +- charts/deckhand/Chart.yaml | 3 +- charts/deckhand/requirements.yaml | 2 +- charts/deps/.gitkeep | 0 deckhand/barbican/driver.py | 4 +- deckhand/common/document.py | 4 +- deckhand/common/utils.py | 2 +- deckhand/control/base.py | 2 +- deckhand/control/buckets.py | 2 +- deckhand/control/middleware.py | 2 +- deckhand/engine/revision_diff.py | 5 +- deckhand/engine/utils.py | 7 +- deckhand/errors.py | 4 +- deckhand/service.py | 4 +- .../revision-deepdiff-success.yaml | 6 +- deckhand/tests/unit/barbican/test_cache.py | 2 +- deckhand/tests/unit/base.py | 4 +- deckhand/tests/unit/common/test_utils.py | 2 +- deckhand/tests/unit/control/base.py | 6 +- .../unit/control/test_api_initialization.py | 4 +- .../unit/control/test_base_controller.py | 2 +- .../unit/control/test_buckets_controller.py | 2 +- deckhand/tests/unit/control/test_errors.py | 4 +- .../tests/unit/control/test_middleware.py | 7 +- .../test_rendered_documents_controller.py | 2 +- .../test_revision_documents_controller.py | 2 +- .../unit/control/test_revisions_controller.py | 2 +- .../test_revisions_rollback_controller.py | 2 +- deckhand/tests/unit/db/test_documents.py | 2 +- .../unit/{fixtures.py => dh_fixtures.py} | 3 +- .../unit/engine/test_document_layering.py | 2 +- ...test_document_layering_and_substitution.py | 2 +- ...ment_layering_and_substitution_negative.py | 2 +- .../engine/test_document_layering_negative.py | 2 +- .../unit/engine/test_document_validation.py | 2 +- .../test_document_validation_negative.py | 2 +- .../tests/unit/engine/test_secrets_manager.py | 2 +- deckhand/tests/unit/test_policy.py | 23 +- doc/requirements.txt | 1 + ....ubuntu_xenial => Dockerfile.ubuntu_focal} | 16 +- requirements.txt | 81 +++---- setup.cfg | 11 +- setup.py | 15 +- test-requirements.txt | 34 +-- .../playbooks/osh-infra-deploy-docker.yaml | 21 +- .../playbooks/osh-infra-upgrade-host.yaml | 4 +- tools/gate/playbooks/vars.yaml | 3 +- .../roles/build-images/defaults/main.yaml | 2 +- .../tasks/build-airship-deckhand-image.yaml | 4 +- .../tasks/deploy-barbican.yaml | 2 +- .../tasks/deploy-deckhand.yaml | 4 + .../tasks/integration-tests.yaml | 2 +- tools/gate/scripts/020-deploy-postgresql.sh | 2 +- tools/helm_install.sh | 2 +- tools/helm_tk.sh | 62 +---- tools/integration-tests.sh | 2 +- tools/run_pifpaf.sh | 2 +- tox.ini | 45 ++-- 61 files changed, 435 insertions(+), 357 deletions(-) create mode 100644 charts/deps/.gitkeep rename deckhand/tests/unit/{fixtures.py => dh_fixtures.py} (99%) rename images/deckhand/{Dockerfile.ubuntu_xenial => Dockerfile.ubuntu_focal} (88%) diff --git a/.dockerignore b/.dockerignore index 14a1bfc2..8d2a2e01 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,121 @@ -.tox -build +# Byte-compiled / optimized / DLL files +**/__pycache__/ +*.py[cod] +*$py.class + +# Unit tests +src/bin/*/tests/unit/ + +# C extensions +*.so + +# Distribution / packaging +.Python +**/env/ +**/build/ +**/develop-eggs/ +**/dist/ +**/downloads/ +**/eggs/ +**/.eggs/ +lib/ +**/lib64/ +parts/ +**/sdist/ +var/ +wheels/ +**/*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +**/htmlcov/ +cov/* +**/.tox/ +**/.coverage +**/.coverage.* +**/.pytest_cache/ +**/.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +doc/_build/ +doc/*/_static/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mypy +.mypy_cache/ + +# Generated bogus docs +ChangeLog +AUTHORS + +# build/lint artifacts +*.tgz +**/*.tgz +/charts/shipyard/charts +/charts/shipyard/requirements.lock +/charts/deps/*/ +.DS_Store + +# vscode +.vscode/ + +!src/lib/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index dc508da3..6ad6eac3 100644 --- a/.gitignore +++ b/.gitignore @@ -111,6 +111,7 @@ ENV/ # makefile build/lint artifacts /charts/deckhand/*.tgz /charts/deckhand/*.lock +/charts/deps/*/ # git Changelog AUTHORS diff --git a/.zuul.yaml b/.zuul.yaml index b4b8ea91..971dd6e4 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -14,52 +14,51 @@ templates: - openstack-cover-jobs - docs-on-readthedocs - - openstack-python36-jobs - - openstack-python3-train-jobs - - openstack-python3-ussuri-jobs + - openstack-python38-jobs vars: rtd_webhook_id: '38572' rtd_project_name: 'airship-deckhand' check: jobs: - - deckhand-tox-py36-postgresql - - deckhand-functional-uwsgi-py36 - - deckhand-functional-docker-py36-ubuntu_xenial - - deckhand-functional-docker-py36-ubuntu_bionic - - deckhand-functional-docker-py36-opensuse - - deckhand-integration-uwsgi-py36 - - deckhand-integration-docker-py36-ubuntu_xenial - - deckhand-integration-docker-py36-ubuntu_bionic - - deckhand-integration-docker-py36-opensuse + - deckhand-tox-py38-postgresql + # Remove in favor of deckhand-functional-docker-py38 it runs the + # same tests just in a container + # - deckhand-functional-uwsgi-py38 + - deckhand-functional-docker-py38-ubuntu_focal + # non-voting, failing, incompatable with updates + # - deckhand-functional-docker-py36-ubuntu_bionic + # non-voting, failing, incompatable with updates + # Remove in favor of deckhand-uwsgi-docker-py38 it runs the + # same tests just in a container + # - deckhand-integration-uwsgi-py38 + - deckhand-integration-docker-py38-ubuntu_focal + # non-voting, failing, incompatable with updates + # - deckhand-integration-docker-py36-ubuntu_bionic - deckhand-chart-build-gate - deckhand-chart-build-latest-htk - - deckhand-docker-build-gate-ubuntu_xenial - - deckhand-docker-build-gate-ubuntu_bionic - - deckhand-docker-build-gate-opensuse + - deckhand-docker-build-gate-ubuntu_focal + # non-voting, failing, incompatable with updates + # - deckhand-docker-build-gate-ubuntu_bionic + - openstack-tox-pep8 - deckhand-airskiff-deployment gate: jobs: - - deckhand-tox-py36-postgresql - - deckhand-functional-docker-py36-ubuntu_xenial + - deckhand-tox-py38-postgresql + - deckhand-functional-docker-py38-ubuntu_focal - deckhand-functional-docker-py36-ubuntu_bionic - - deckhand-functional-docker-py36-opensuse - - deckhand-integration-docker-py36-ubuntu_xenial + - deckhand-integration-docker-py38-ubuntu_focal - deckhand-integration-docker-py36-ubuntu_bionic - - deckhand-integration-docker-py36-opensuse - deckhand-chart-build-gate - - deckhand-docker-build-gate-ubuntu_xenial + - deckhand-docker-build-gate-ubuntu_focal - deckhand-docker-build-gate-ubuntu_bionic - - deckhand-docker-build-gate-opensuse - openstack-tox-pep8 post: jobs: - deckhand-upload-git-mirror - - deckhand-docker-publish-ubuntu_xenial + - deckhand-docker-publish-ubuntu_focal - deckhand-docker-publish-ubuntu_bionic - - deckhand-docker-publish-opensuse - - deckhand-docker-tag-ubuntu_xenial + - deckhand-docker-tag-ubuntu_focal - deckhand-docker-tag-ubuntu_bionic - - deckhand-docker-tag-opensuse - nodeset: name: deckhand-single-node @@ -73,22 +72,36 @@ - name: primary label: ubuntu-bionic +- nodeset: + name: deckhand-single-node-focal + nodes: + - name: primary + label: ubuntu-focal + +- nodeset: + name: deckhand-single-node-jammy + nodes: + - name: primary + label: ubuntu-jammy + - job: - name: deckhand-tox-py36-postgresql - parent: openstack-tox-py36 + name: deckhand-tox-py38-postgresql + parent: openstack-tox-py38 + nodeset: deckhand-single-node-focal pre-run: - tools/gate/playbooks/install-postgresql.yaml vars: - tox_envlist: py36-postgresql + tox_envlist: py38-postgresql - job: - name: deckhand-functional-uwsgi-py36 + name: deckhand-functional-uwsgi-py38 + voting: false description: | Run tox-based functional tests for the Airship Deckhand project using a minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf - for ephemeral PostgreSQL DB, under cPython version 3.6. + for ephemeral PostgreSQL DB, under cPython version 3.8. run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal vars: tox_envlist: functional-dev irrelevant-files: &irrelevant-files @@ -104,6 +117,7 @@ description: | Base job for running deckhand functional tests. Runs tests against Docker image generated from source code. + nodeset: deckhand-single-node-focal roles: - zuul: openstack/openstack-helm-infra timeout: 3600 @@ -115,22 +129,22 @@ - openstack/openstack-helm-infra - job: - name: deckhand-functional-docker-py36-ubuntu_xenial - voting: false + name: deckhand-functional-docker-py38-ubuntu_focal description: | Run tox-based functional tests for the Airship Deckhand project under - cPython version 3.6. Uses tox with the ``functional-py36`` environment. - Ubuntu (xenial) image is built and used. + cPython version 3.8. Uses tox with the ``functional-py38`` environment. + Ubuntu (focal) image is built and used. parent: deckhand-functional-docker-base - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal vars: tox_envlist: functional disable_keystone: true - distro: ubuntu_xenial + distro: ubuntu_focal irrelevant-files: *irrelevant-files - job: name: deckhand-functional-docker-py36-ubuntu_bionic + voting: false description: | Run tox-based functional tests for the Airship Deckhand project under cPython version 3.6. Uses tox with the ``functional-py36`` environment. @@ -144,29 +158,15 @@ irrelevant-files: *irrelevant-files - job: - name: deckhand-functional-docker-py36-opensuse + name: deckhand-integration-uwsgi-py38 voting: false - description: | - Run tox-based functional tests for the Airship Deckhand project under - cPython version 3.6. Uses tox with the ``functional-py36`` environment. - Opensuse image is built and used. - parent: deckhand-functional-docker-base - nodeset: deckhand-single-node - vars: - tox_envlist: functional - disable_keystone: true - distro: opensuse_15 - irrelevant-files: *irrelevant-files - -- job: - name: deckhand-integration-uwsgi-py36 description: | Run tox-based integration tests for the Airship Deckhand project using a minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf - for ephemeral PostgreSQL DB, under cPython version 3.6. + for ephemeral PostgreSQL DB, under cPython version 3.8. timeout: 3600 run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal irrelevant-files: *irrelevant-files vars: disable_keystone: true @@ -177,7 +177,7 @@ Build charts using pinned Helm toolkit. timeout: 900 run: tools/gate/playbooks/build-charts.yaml - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal - job: name: deckhand-chart-build-latest-htk @@ -185,7 +185,7 @@ Build charts using latest Helm toolkit. timeout: 900 run: tools/gate/playbooks/build-charts.yaml - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal vars: HTK_COMMIT: master @@ -194,6 +194,7 @@ description: | Base job for running deckhand integration tests. Runs tests against Docker image generated from source code. + nodeset: deckhand-single-node-focal timeout: 3600 roles: - zuul: openstack/openstack-helm-infra @@ -203,7 +204,8 @@ pre-run: - tools/gate/playbooks/osh-infra-upgrade-host.yaml - tools/gate/playbooks/osh-infra-deploy-docker.yaml - run: tools/gate/playbooks/run-integration-tests-docker.yaml + run: + - tools/gate/playbooks/run-integration-tests-docker.yaml post-run: tools/gate/playbooks/osh-infra-collect-logs.yaml required-projects: - openstack/openstack-helm @@ -214,16 +216,15 @@ - ^releasenotes/.*$ - job: - name: deckhand-integration-docker-py36-ubuntu_xenial - voting: false + name: deckhand-integration-docker-py38-ubuntu_focal description: | Run tox-based integration tests for the Airship Deckhand project under - cPython version 3.6. Builds ubuntu (xenial) deckhand image. + cPython version 3.8. Builds ubuntu (focal) deckhand image. parent: deckhand-integration-docker-base nodeset: openstack-helm-single-node vars: disable_keystone: false - distro: ubuntu_xenial + distro: ubuntu_focal - job: name: deckhand-integration-docker-py36-ubuntu_bionic @@ -259,20 +260,7 @@ - ^releasenotes/.*$ - job: - name: deckhand-integration-docker-py36-opensuse - voting: false - description: | - Run tox-based integration tests for the Airship Deckhand project under - cPython version 3.6. Builds opensuse deckhand image. - parent: deckhand-integration-docker-base - nodeset: openstack-helm-single-node - vars: - disable_keystone: false - distro: opensuse_15 - -- job: - name: deckhand-docker-build-gate-ubuntu_xenial - voting: false + name: deckhand-docker-build-gate-ubuntu_focal timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml nodeset: deckhand-single-node @@ -285,13 +273,14 @@ - ^setup.cfg$ vars: publish: false - distro: ubuntu_xenial + distro: ubuntu_focal tags: dynamic: patch_set: true - job: name: deckhand-docker-build-gate-ubuntu_bionic + voting: false timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml nodeset: deckhand-single-node @@ -304,35 +293,20 @@ patch_set: true - job: - name: deckhand-docker-build-gate-opensuse - voting: false - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node - irrelevant-files: *non-code-files-template - vars: - publish: false - distro: opensuse_15 - tags: - dynamic: - patch_set: true - -- job: - name: deckhand-docker-publish-ubuntu_xenial - voting: false + name: deckhand-docker-publish-ubuntu_focal description: | Runs on every merge, unless files in a dictionary below are changed. Builds and publishes container ubuntu images on quay.io with a set of tags listed in vars section. Waits in Zuul queue for a node (VM) assignment. timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal secrets: - airship_deckhand_quay_creds irrelevant-files: *non-code-files-template vars: publish: true - distro: ubuntu_xenial + distro: ubuntu_focal tags: dynamic: branch: true @@ -342,6 +316,7 @@ - job: name: deckhand-docker-publish-ubuntu_bionic + voting: false description: | Runs on every merge, unless files in a dictionary below are changed. Builds and publishes container ubuntu images on quay.io with a set of tags @@ -363,31 +338,7 @@ - latest - job: - name: deckhand-docker-publish-opensuse - voting: false - description: | - Runs on every merge, unless files in a dictionary below are changed. - Builds and publishes container opensuse images on quay.io with a set of tags - listed in vars section. Waits in Zuul queue for a node (VM) assignment. - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node - secrets: - - airship_deckhand_quay_creds - irrelevant-files: *non-code-files-template - vars: - publish: true - distro: opensuse_15 - tags: - dynamic: - branch: true - commit: true - static: - - latest - -- job: - name: deckhand-docker-tag-ubuntu_xenial - voting: false + name: deckhand-docker-tag-ubuntu_focal description: | Runs on every merge when files in a dictionalry below are changed, and adds git commit id tag onto the ubuntu container image published on quay.io, @@ -400,10 +351,11 @@ secrets: - airship_deckhand_quay_creds vars: - distro: ubuntu_xenial + distro: ubuntu_focal - job: name: deckhand-docker-tag-ubuntu_bionic + voting: false description: | Runs on every merge when files in a dictionalry below are changed, and adds git commit id tag onto the ubuntu container image published on quay.io, @@ -418,27 +370,6 @@ vars: distro: ubuntu_bionic -- job: - name: deckhand-docker-tag-opensuse - voting: false - description: | - Runs on every merge when files in a dictionalry below are changed, and - adds git commit id tag onto the opensuse container image published on quay.io, - which has `latest` tag set. Does not wait in queue for a node (VM) - assignment, runs almost immediately. - timeout: 1800 - run: tools/gate/playbooks/docker-image-tag.yaml - nodeset: - nodes: [] - secrets: - - airship_deckhand_quay_creds - vars: - distro: opensuse_15 -# file pattern here must be exactly the same as in -# deckhand-docker-publish job above, -# job will be executed on merge only when any of this files get changed - files: *non-code-files-template - - secret: name: airship_deckhand_quay_creds data: diff --git a/Makefile b/Makefile index 498bb281..c3b7f280 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ USE_PROXY ?= false PUSH_IMAGE ?= false # use this variable for image labels added in internal build process LABEL ?= org.airshipit.build=community -DISTRO ?= ubuntu_bionic +DISTRO ?= ubuntu_focal COMMIT ?= $(shell git rev-parse HEAD) IMAGE := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} @@ -37,13 +37,13 @@ images: build_deckhand # Create tgz of the chart .PHONY: charts -charts: helm-init +charts: helm-toolkit $(HELM) dep up charts/deckhand $(HELM) package charts/deckhand # Initialize local helm config -.PHONY: helm-init -helm-init: helm-install +.PHONY: helm-toolkit +helm-toolkit: helm-install tools/helm_tk.sh $(HELM) # Install helm binary diff --git a/charts/deckhand/Chart.yaml b/charts/deckhand/Chart.yaml index 87223af6..298bc7ed 100644 --- a/charts/deckhand/Chart.yaml +++ b/charts/deckhand/Chart.yaml @@ -15,7 +15,8 @@ apiVersion: v1 description: A Helm chart for Deckhand name: deckhand -version: 0.1.0 +version: 0.2.0 +appVersion: 1.0 keywords: - deckhand home: https://github.com/openstack/airship-deckhand diff --git a/charts/deckhand/requirements.yaml b/charts/deckhand/requirements.yaml index c81c2135..f0eabffd 100644 --- a/charts/deckhand/requirements.yaml +++ b/charts/deckhand/requirements.yaml @@ -14,5 +14,5 @@ dependencies: - name: helm-toolkit - repository: http://localhost:8879/charts + repository: file://../deps/helm-toolkit version: ">= 0.1.0" diff --git a/charts/deps/.gitkeep b/charts/deps/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/deckhand/barbican/driver.py b/deckhand/barbican/driver.py index 83dc9291..b269ef73 100644 --- a/deckhand/barbican/driver.py +++ b/deckhand/barbican/driver.py @@ -163,7 +163,7 @@ class BarbicanDriver(object): raise errors.BarbicanServerException(details=str(e)) payload = secret.payload - if secret.secret_type == 'opaque': + if secret.secret_type == 'opaque': # nosec LOG.debug('Base64-decoding original payload ' 'for document [%s, %s] %s.', *src_doc.meta) secret = self._base64_decode_payload(payload) @@ -182,7 +182,7 @@ class BarbicanDriver(object): except (barbicanclient.exceptions.HTTPAuthError, barbicanclient.exceptions.HTTPServerError) as e: LOG.exception(str(e)) - raise errors.BarbicanException(details=str(e)) + raise errors.BarbicanClientException(details=str(e)) except barbicanclient.exceptions.HTTPClientError as e: if e.status_code == 404: LOG.warning('Could not delete secret %s because it was not ' diff --git a/deckhand/common/document.py b/deckhand/common/document.py index 6d7f9155..86072a4c 100644 --- a/deckhand/common/document.py +++ b/deckhand/common/document.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections +from collections.abc import Iterable import re import hashlib @@ -167,7 +167,7 @@ class DocumentDict(dict): :param documents: Documents to wrap in this class. :type documents: iterable """ - if not isinstance(documents, collections.Iterable): + if not isinstance(documents, Iterable): documents = [documents] return [DocumentDict(d) for d in documents] diff --git a/deckhand/common/utils.py b/deckhand/common/utils.py index 2516e6ed..9de3747c 100644 --- a/deckhand/common/utils.py +++ b/deckhand/common/utils.py @@ -49,7 +49,7 @@ def to_camel_case(s): def to_snake_case(name): """Convert string to snake case.""" - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', str(name)) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() diff --git a/deckhand/control/base.py b/deckhand/control/base.py index 897db2ed..b15a6908 100644 --- a/deckhand/control/base.py +++ b/deckhand/control/base.py @@ -55,7 +55,7 @@ class BaseResource(object): :param allow_empty: Whether the request body can be empty. :returns: List of dicts if ``expect_list`` is True or else a dict. """ - raw_data = req.stream.read(req.content_length or 0) + raw_data = req.bounded_stream.read() if not allow_empty and not raw_data: error_msg = ("The request body must not be empty.") diff --git a/deckhand/control/buckets.py b/deckhand/control/buckets.py index dc531966..2d58bb62 100644 --- a/deckhand/control/buckets.py +++ b/deckhand/control/buckets.py @@ -64,7 +64,7 @@ class BucketsResource(api_base.BaseResource): created_documents = self._create_revision_documents( bucket_name, documents) - resp.body = self.view_builder.list(created_documents) + resp.text = self.view_builder.list(created_documents) resp.status = falcon.HTTP_200 def _encrypt_secret_documents(self, documents): diff --git a/deckhand/control/middleware.py b/deckhand/control/middleware.py index 7acd8c10..c5cb3127 100644 --- a/deckhand/control/middleware.py +++ b/deckhand/control/middleware.py @@ -161,7 +161,7 @@ class YAMLTranslator(HookableMiddlewareMixin, object): ) raise falcon.HTTPUnsupportedMediaType(description=message) - def process_response(self, req, resp, resource): + def process_response(self, req, resp, resource, req_succeeded): """Converts responses to ``application/x-yaml`` content type.""" if resp.status != '204 No Content': resp.set_header('Content-Type', 'application/x-yaml') diff --git a/deckhand/engine/revision_diff.py b/deckhand/engine/revision_diff.py index e218630e..f8373c1c 100644 --- a/deckhand/engine/revision_diff.py +++ b/deckhand/engine/revision_diff.py @@ -248,7 +248,8 @@ def _diff_buckets(b1, b2): # serializing to json then deserializing # to dict. data_changed = jsonpickle.decode( - DeepDiff(d['data'], b2_tmp[k]['data']).json) + DeepDiff(d['data'], b2_tmp[k]['data']).to_json()) + # deepdiff doesn't provide custom exceptions; # have to use Exception. except Exception as ex: @@ -256,7 +257,7 @@ def _diff_buckets(b1, b2): try: metadata_changed = jsonpickle.decode( DeepDiff(d['metadata'], - b2_tmp[k]['metadata']).json) + b2_tmp[k]['metadata']).to_json()) except Exception as ex: raise errors.DeepDiffException(details=str(ex)) diff --git a/deckhand/engine/utils.py b/deckhand/engine/utils.py index 32e85eb6..11562cf8 100644 --- a/deckhand/engine/utils.py +++ b/deckhand/engine/utils.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping from deckhand.common import utils @@ -32,7 +35,7 @@ def deep_merge(dct, merge_dct): """ for k, v in merge_dct.items(): if (k in dct and isinstance(dct[k], dict) and - isinstance(merge_dct[k], collections.Mapping)): + isinstance(merge_dct[k], Mapping)): deep_merge(dct[k], merge_dct[k]) else: dct[k] = merge_dct[k] diff --git a/deckhand/errors.py b/deckhand/errors.py index 9d317247..a9e0bbdd 100644 --- a/deckhand/errors.py +++ b/deckhand/errors.py @@ -102,10 +102,10 @@ def format_error_resp(req, } resp.status = status_code - resp.body = yaml.safe_dump(error_response) + resp.text = yaml.safe_dump(error_response) -def default_exception_handler(ex, req, resp, params): +def default_exception_handler(req, resp, ex, params): """Catch-all exception handler for standardized output. If this is a standard falcon HTTPError, rethrow it for handling by diff --git a/deckhand/service.py b/deckhand/service.py index e1137adf..a514c6f9 100644 --- a/deckhand/service.py +++ b/deckhand/service.py @@ -17,7 +17,7 @@ import os import falcon from oslo_config import cfg from oslo_log import log -from werkzeug.contrib.profiler import ProfilerMiddleware +from werkzeug.middleware.profiler import ProfilerMiddleware from deckhand.control import base from deckhand.control import buckets @@ -88,7 +88,7 @@ def deckhand_app_factory(global_config, **local_config): middleware.ContextMiddleware(), middleware.LoggingMiddleware()] - app = falcon.API(request_type=base.DeckhandRequest, + app = falcon.App(request_type=base.DeckhandRequest, middleware=middleware_list) if CONF.profiler: LOG.warning("Profiler ENABLED. Expect significant " diff --git a/deckhand/tests/functional/gabbits/revision-deepdiff/revision-deepdiff-success.yaml b/deckhand/tests/functional/gabbits/revision-deepdiff/revision-deepdiff-success.yaml index 18e3ab25..9d8497fe 100644 --- a/deckhand/tests/functional/gabbits/revision-deepdiff/revision-deepdiff-success.yaml +++ b/deckhand/tests/functional/gabbits/revision-deepdiff/revision-deepdiff-success.yaml @@ -241,7 +241,8 @@ tests: $.[0].'bucket_a diff'.document_changed.details: ('example/Kind/v1', 'doc-a'): data_changed: - dictionary_item_added: !!set {"root['foo']"} + dictionary_item_added: + ["root['foo']"] metadata_changed: {} - name: removing_key_in_doc-a @@ -273,7 +274,8 @@ tests: $.[0].'bucket_a diff'.document_changed.details: ('example/Kind/v1', 'doc-a'): data_changed: - dictionary_item_removed: !!set {"root['foo']"} + dictionary_item_removed: + ["root['foo']"] metadata_changed: {} - name: verify_invalid_input diff --git a/deckhand/tests/unit/barbican/test_cache.py b/deckhand/tests/unit/barbican/test_cache.py index 1321aaea..0b32483f 100644 --- a/deckhand/tests/unit/barbican/test_cache.py +++ b/deckhand/tests/unit/barbican/test_cache.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import testtools from deckhand.barbican import cache diff --git a/deckhand/tests/unit/base.py b/deckhand/tests/unit/base.py index 085bb6cf..9b637614 100644 --- a/deckhand/tests/unit/base.py +++ b/deckhand/tests/unit/base.py @@ -17,7 +17,7 @@ from __future__ import absolute_import import os import fixtures -import mock +from unittest import mock from oslo_config import cfg from oslo_log import log as logging import testtools @@ -26,7 +26,7 @@ from deckhand.conf import config # noqa: Calls register_opts(CONF) from deckhand.db.sqlalchemy import api as db_api from deckhand.engine import cache from deckhand.tests import test_utils -from deckhand.tests.unit import fixtures as dh_fixtures +from deckhand.tests.unit import dh_fixtures CONF = cfg.CONF logging.register_options(CONF) diff --git a/deckhand/tests/unit/common/test_utils.py b/deckhand/tests/unit/common/test_utils.py index 238745ac..f31303de 100644 --- a/deckhand/tests/unit/common/test_utils.py +++ b/deckhand/tests/unit/common/test_utils.py @@ -14,7 +14,7 @@ import hashlib import jsonpath_ng -import mock +from unittest import mock from oslo_serialization import jsonutils as json from testtools.matchers import Equals diff --git a/deckhand/tests/unit/control/base.py b/deckhand/tests/unit/control/base.py index 34dc7448..822d5425 100644 --- a/deckhand/tests/unit/control/base.py +++ b/deckhand/tests/unit/control/base.py @@ -19,7 +19,7 @@ from falcon import testing as falcon_testing from deckhand import service from deckhand.tests.unit import base as test_base -from deckhand.tests.unit import fixtures +from deckhand.tests.unit import dh_fixtures class BaseControllerTest(test_base.DeckhandWithDBTestCase, @@ -30,10 +30,10 @@ class BaseControllerTest(test_base.DeckhandWithDBTestCase, super(BaseControllerTest, self).setUp() self.app = falcon_testing.TestClient( service.deckhand_app_factory(None)) - self.policy = self.useFixture(fixtures.RealPolicyFixture()) + self.policy = self.useFixture(dh_fixtures.RealPolicyFixture()) # NOTE: development_mode allows these unit tests to get around # Keystone authentication. - self.useFixture(fixtures.ConfPatcher(development_mode=True)) + self.useFixture(dh_fixtures.ConfPatcher(development_mode=True)) def _read_data(self, file_name): # Reads data from a file in the resources directory diff --git a/deckhand/tests/unit/control/test_api_initialization.py b/deckhand/tests/unit/control/test_api_initialization.py index cddfccda..b5bd875f 100644 --- a/deckhand/tests/unit/control/test_api_initialization.py +++ b/deckhand/tests/unit/control/test_api_initialization.py @@ -15,7 +15,7 @@ import inspect import os -import mock +from unittest import mock from deckhand.common import utils from deckhand.control import api @@ -72,7 +72,7 @@ class TestApi(test_base.DeckhandTestCase): @mock.patch('deckhand.service.falcon', autospec=True) def test_init_application(self, mock_falcon, mock_logging, mock_db_api, _): - mock_falcon_api = mock_falcon.API.return_value + mock_falcon_api = mock_falcon.App.return_value self.override_config( 'connection', mock.sentinel.db_connection, group='database') diff --git a/deckhand/tests/unit/control/test_base_controller.py b/deckhand/tests/unit/control/test_base_controller.py index 6f57dc6a..4c58bdea 100644 --- a/deckhand/tests/unit/control/test_base_controller.py +++ b/deckhand/tests/unit/control/test_base_controller.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock from deckhand.control import base as api_base from deckhand.tests.unit.control import base as test_base diff --git a/deckhand/tests/unit/control/test_buckets_controller.py b/deckhand/tests/unit/control/test_buckets_controller.py index 27a234b4..581e1b9a 100644 --- a/deckhand/tests/unit/control/test_buckets_controller.py +++ b/deckhand/tests/unit/control/test_buckets_controller.py @@ -14,7 +14,7 @@ import yaml -import mock +from unittest import mock from oslo_config import cfg from deckhand.engine import secrets_manager diff --git a/deckhand/tests/unit/control/test_errors.py b/deckhand/tests/unit/control/test_errors.py index 538973a0..b06e18e5 100644 --- a/deckhand/tests/unit/control/test_errors.py +++ b/deckhand/tests/unit/control/test_errors.py @@ -16,7 +16,7 @@ import os import yaml import falcon -import mock +from unittest import mock from deckhand.common import document as document_wrapper from deckhand import policy @@ -58,7 +58,7 @@ class TestErrorFormatting(test_base.BaseControllerTest): 'message': 'Unhandled Exception raised: test error', 'metadata': {} } - body = yaml.load(resp.text) + body = yaml.safe_load(resp.text) self.assertEqual(500, resp.status_code) self.assertEqual(expected, body) diff --git a/deckhand/tests/unit/control/test_middleware.py b/deckhand/tests/unit/control/test_middleware.py index 55fa862d..dea3f13f 100644 --- a/deckhand/tests/unit/control/test_middleware.py +++ b/deckhand/tests/unit/control/test_middleware.py @@ -14,7 +14,7 @@ import yaml -import mock +from unittest import mock from deckhand import factories from deckhand.tests.unit.control import base as test_base @@ -118,16 +118,17 @@ class TestYAMLTranslatorNegative(test_base.BaseControllerTest): 'errorType': 'HTTPMissingHeader', 'messageList': [{ 'error': True, - 'message': "The Content-Type header is required." + 'message': "The \"Content-Type\" header is required." }] }, 'kind': 'Status', - 'message': "The Content-Type header is required.", + 'message': "The \"Content-Type\" header is required.", 'metadata': {}, 'reason': 'Unspecified', 'retry': False, 'status': 'Failure' } + self.assertEqual(expected, yaml.safe_load(resp.content)) def test_request_with_invalid_content_type_raises_exception(self): diff --git a/deckhand/tests/unit/control/test_rendered_documents_controller.py b/deckhand/tests/unit/control/test_rendered_documents_controller.py index 9a022c21..1ab75a1b 100644 --- a/deckhand/tests/unit/control/test_rendered_documents_controller.py +++ b/deckhand/tests/unit/control/test_rendered_documents_controller.py @@ -15,7 +15,7 @@ import re import yaml -import mock +from unittest import mock from deckhand.common.document import DocumentDict as dd from deckhand.control import revision_documents diff --git a/deckhand/tests/unit/control/test_revision_documents_controller.py b/deckhand/tests/unit/control/test_revision_documents_controller.py index f8d0343d..c1abcc15 100644 --- a/deckhand/tests/unit/control/test_revision_documents_controller.py +++ b/deckhand/tests/unit/control/test_revision_documents_controller.py @@ -14,7 +14,7 @@ import yaml -import mock +from unittest import mock from deckhand.common.document import DocumentDict as document_dict from deckhand.engine import secrets_manager diff --git a/deckhand/tests/unit/control/test_revisions_controller.py b/deckhand/tests/unit/control/test_revisions_controller.py index cc38c45f..f191e6e5 100644 --- a/deckhand/tests/unit/control/test_revisions_controller.py +++ b/deckhand/tests/unit/control/test_revisions_controller.py @@ -14,7 +14,7 @@ import os -import mock +from unittest import mock import yaml from deckhand.engine import secrets_manager diff --git a/deckhand/tests/unit/control/test_revisions_rollback_controller.py b/deckhand/tests/unit/control/test_revisions_rollback_controller.py index 0ada8e11..49d43852 100644 --- a/deckhand/tests/unit/control/test_revisions_rollback_controller.py +++ b/deckhand/tests/unit/control/test_revisions_rollback_controller.py @@ -14,7 +14,7 @@ import yaml -import mock +from unittest import mock from deckhand.engine import secrets_manager from deckhand import factories diff --git a/deckhand/tests/unit/db/test_documents.py b/deckhand/tests/unit/db/test_documents.py index 246c4950..a05ef3c9 100644 --- a/deckhand/tests/unit/db/test_documents.py +++ b/deckhand/tests/unit/db/test_documents.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import testtools from deckhand.db.sqlalchemy import api as db_api diff --git a/deckhand/tests/unit/fixtures.py b/deckhand/tests/unit/dh_fixtures.py similarity index 99% rename from deckhand/tests/unit/fixtures.py rename to deckhand/tests/unit/dh_fixtures.py index 45e162bd..e8ee7ff2 100644 --- a/deckhand/tests/unit/fixtures.py +++ b/deckhand/tests/unit/dh_fixtures.py @@ -22,7 +22,7 @@ import os import yaml import fixtures -import mock +from unittest import mock from oslo_config import cfg from oslo_policy import opts as policy_opts from oslo_policy import policy as oslo_policy @@ -41,7 +41,6 @@ class ConfPatcher(fixtures.Fixture): it's teardown. """ - def __init__(self, **kwargs): """Constructor diff --git a/deckhand/tests/unit/engine/test_document_layering.py b/deckhand/tests/unit/engine/test_document_layering.py index 1fbf1fbf..464f7dac 100644 --- a/deckhand/tests/unit/engine/test_document_layering.py +++ b/deckhand/tests/unit/engine/test_document_layering.py @@ -14,7 +14,7 @@ import yaml -import mock +from unittest import mock from deckhand.engine import layering from deckhand.engine import secrets_manager diff --git a/deckhand/tests/unit/engine/test_document_layering_and_substitution.py b/deckhand/tests/unit/engine/test_document_layering_and_substitution.py index 5ccf58b7..3ccf88ea 100644 --- a/deckhand/tests/unit/engine/test_document_layering_and_substitution.py +++ b/deckhand/tests/unit/engine/test_document_layering_and_substitution.py @@ -14,7 +14,7 @@ import itertools -import mock +from unittest import mock import yaml from deckhand import factories diff --git a/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py b/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py index 2215c015..5c819ff0 100644 --- a/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py +++ b/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock from deckhand.engine import layering from deckhand import errors diff --git a/deckhand/tests/unit/engine/test_document_layering_negative.py b/deckhand/tests/unit/engine/test_document_layering_negative.py index 85a7f42e..04f8ba21 100644 --- a/deckhand/tests/unit/engine/test_document_layering_negative.py +++ b/deckhand/tests/unit/engine/test_document_layering_negative.py @@ -14,7 +14,7 @@ import copy -import mock +from unittest import mock from deckhand.engine import layering from deckhand import errors diff --git a/deckhand/tests/unit/engine/test_document_validation.py b/deckhand/tests/unit/engine/test_document_validation.py index 180ecd07..9a0a2c51 100644 --- a/deckhand/tests/unit/engine/test_document_validation.py +++ b/deckhand/tests/unit/engine/test_document_validation.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock from deckhand.common import utils from deckhand.engine import document_validation diff --git a/deckhand/tests/unit/engine/test_document_validation_negative.py b/deckhand/tests/unit/engine/test_document_validation_negative.py index 82c430ef..37b774d2 100644 --- a/deckhand/tests/unit/engine/test_document_validation_negative.py +++ b/deckhand/tests/unit/engine/test_document_validation_negative.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock from deckhand.engine import document_validation from deckhand import errors diff --git a/deckhand/tests/unit/engine/test_secrets_manager.py b/deckhand/tests/unit/engine/test_secrets_manager.py index 4a3bdf0e..c521e204 100644 --- a/deckhand/tests/unit/engine/test_secrets_manager.py +++ b/deckhand/tests/unit/engine/test_secrets_manager.py @@ -15,7 +15,7 @@ import copy import yaml -import mock +from unittest import mock from oslo_serialization import base64 from oslo_utils import uuidutils import testtools diff --git a/deckhand/tests/unit/test_policy.py b/deckhand/tests/unit/test_policy.py index cccfcd6c..198736d5 100644 --- a/deckhand/tests/unit/test_policy.py +++ b/deckhand/tests/unit/test_policy.py @@ -13,13 +13,14 @@ # limitations under the License. import falcon -import mock +from falcon import testing as falcon_testing +from unittest import mock from oslo_policy import policy as common_policy from deckhand.control import base as api_base import deckhand.policy from deckhand.tests.unit import base as test_base -from deckhand.tests.unit import fixtures +from deckhand.tests.unit import dh_fixtures class PolicyBaseTestCase(test_base.DeckhandTestCase): @@ -34,7 +35,7 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase): "deckhand:list_cleartext_documents": [['rule:admin_api']] } - self.useFixture(fixtures.RealPolicyFixture(False)) + self.useFixture(dh_fixtures.RealPolicyFixture(False)) self._set_rules() def _set_rules(self): @@ -51,9 +52,23 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase): noop(*api_args) def _get_args(self): + env = falcon_testing.create_environ( + path='/', + query_string='', + scheme='http', + host='falconframework.org', + port=None, + headers={'Content-Type': 'application/x-yaml'}, + app='', + body='', + method='POST', + wsgierrors=None, + file_wrapper=None) + + falcon_req = api_base.DeckhandRequest(env) + # Returns the first two arguments that would be passed to any falcon # on_{HTTP_VERB} method: (self (which is mocked), falcon Request obj). - falcon_req = api_base.DeckhandRequest(mock.MagicMock()) return (mock.Mock(), falcon_req) diff --git a/doc/requirements.txt b/doc/requirements.txt index ba09cc72..916fb8ad 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -4,6 +4,7 @@ sphinx>=1.6.2 # BSD sphinx_rtd_theme reno>=2.5.0 # Apache-2.0 +pylibyaml==0.1.0 plantuml sphinxcontrib-apidoc>=0.2.0 # BSD sphinxcontrib-plantuml diff --git a/images/deckhand/Dockerfile.ubuntu_xenial b/images/deckhand/Dockerfile.ubuntu_focal similarity index 88% rename from images/deckhand/Dockerfile.ubuntu_xenial rename to images/deckhand/Dockerfile.ubuntu_focal index 75341d6a..2fb34cdb 100644 --- a/images/deckhand/Dockerfile.ubuntu_xenial +++ b/images/deckhand/Dockerfile.ubuntu_focal @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG FROM=ubuntu:16.04 +ARG FROM=ubuntu:20.04 FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' @@ -51,7 +51,7 @@ RUN set -x && \ python3-pip \ python3-setuptools \ --no-install-recommends \ - && python3 -m pip install -U 'pip<21.0' \ + && python3 -m pip install -U pip \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ @@ -88,8 +88,16 @@ RUN chown -R deckhand: /home/deckhand \ # Set work directory and install dependencies WORKDIR /home/deckhand -RUN pip3 install -r requirements.txt -RUN python3 setup.py install +RUN pip3 install --no-cache-dir -r requirements.txt + +# Setting deckhand version for BPR +ENV PBR_VERSION 1.0 + + +COPY . /opt/deckhand +RUN pip3 install -e /opt/deckhand \ + && echo "/opt/deckhand" \ + > /usr/local/lib/python3.8/dist-packages/deckhand.pth # Set user to deckhand USER deckhand diff --git a/requirements.txt b/requirements.txt index 35f2b0dd..582b8c26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,51 +2,54 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=3.0.1,<3.1.0 - -alembic==1.0.1 -amqp<2.7,>=2.6.0 -beaker==1.10.0 -cryptography>=2.7 -deepdiff==3.3.0 -falcon==1.4.1 -jsonpath-ng==1.4.3 -jsonschema>=3.0.1,<4 -keystoneauth1>=3.18.0 -keystonemiddleware==5.3.0 -kombu<4.7,>=4.6.10 -networkx==2.2 -oslo.cache==1.38.1 -oslo.concurrency==3.28.1 -oslo.config==7.0.0 -oslo.context>=2.21.0 -oslo.messaging==9.1.1 -oslo.db==4.41.1 -oslo.log==3.45.2 -oslo.middleware==3.36.0 -oslo.policy==1.40.1 -oslo.serialization==2.29.2 -oslo.utils==3.42.1 -pbr==5.4.5 -PasteDeploy==1.5.2 -Paste==3.0.1 -psycopg2-binary==2.8.4 -pylibyaml~=0.1 -pyyaml~=5.1 -python-dateutil>=2.8.1 +hacking==3.0.1 +SQLAlchemy==1.4.23 +alembic==1.7.1 +amqp==5.0.8 +Beaker==1.12.0 +cryptography==3.4.8 +deepdiff==5.8.1 +falcon==3.1.1 +jsonpath-ng==1.5.3 +jsonschema==3.2.0 +keystoneauth1==5.1.1 +keystonemiddleware==10.2.0 +kombu==5.1.0 +networkx==2.6.2 +oslo.cache==2.8.2 +oslo.concurrency==4.4.1 +oslo.config==8.7.1 +oslo.context==5.0.0 +oslo.messaging==12.9.4 +oslo.db==11.0.0 +oslo.log==4.6.0 +oslo.middleware==4.4.0 +oslo.policy==4.0.0 +oslo.serialization==4.2.0 +oslo.utils==4.10.2 +pbr==5.6.0 +PasteDeploy==3.0.1 +Paste==3.5.0 +psycopg2-binary==2.9.5 +pylibyaml==0.1.0 +PyYAML==5.4.1 +python-dateutil==2.8.2 # TODO(alanmeadows) # this must match the container service # likely this should be imported from a # container sidecar long-term -python-barbicanclient==4.7.0 +python-barbicanclient==5.2.0 python-keystoneclient==3.22.0 python-memcached==1.59 -Routes==2.4.1 -six>=1.15.0 -stevedore>=1.30.0 -urllib3==1.25.9 -uwsgi~=2.0.19.1 +Routes==2.5.1 +six==1.16.0 +sphinx-rtd-theme==1.1.1 +stestr==3.2.0 +stevedore==4.1.1 +urllib3==1.26.6 +uWSGI==2.0.21 # To support profiling in non-prod -Werkzeug==0.16.1 +Werkzeug==2.0.1 +jsonpickle \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 7347ea3e..4a9d9ea6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,11 @@ [metadata] name = Deckhand +version = 1.0 summary = Storage service for YAML-based configuration documents, which are managed through version control and automatically validated. -description-file = README.rst +description_file = README.rst author = The Airship Authors -author-email = airship-discuss@lists.airshipit.org -home-page = https://airship-deckhand.readthedocs.io/ +author_email = airship-discuss@lists.airshipit.org +home_page = https://airship-deckhand.readthedocs.io/ classifier = Intended Audience :: Information Technology @@ -13,8 +14,8 @@ classifier = Operating System :: POSIX :: Linux Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.10 [files] packages = diff --git a/setup.py b/setup.py index 0b220f45..ffec5116 100644 --- a/setup.py +++ b/setup.py @@ -12,16 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import setuptools +from setuptools import setup -# In python < 2.7.4, a lazy loading of package `pbr` will break -# setuptools if some other modules registered functions in `atexit`. -# solution from: http://bugs.python.org/issue15881#msg170215 -try: - import multiprocessing # noqa -except ImportError: - pass - -setuptools.setup( +setup( setup_requires=['pbr>=2.0.0'], - pbr=True) + pbr=True +) diff --git a/test-requirements.txt b/test-requirements.txt index 1034c706..7f46f956 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,22 +2,28 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -amqp<2.7,>=2.6.0 -coverage==4.5.1 +amqp==5.0.8 +coverage==5.5 fixtures==3.0.0 -python-subunit>=1.4.0 -os-testr==1.0.0 +python-subunit==1.4.0 +os-testr==2.0.1 testrepository==0.0.20 -testtools==2.3.0 -bandit==1.6.2 +testtools==2.5.0 +bandit==1.7.4 # NOTE(felipemonteiro): Pin here because later versions require that # content-type be present in empty responses. gabbi==1.35.1 -pifpaf==2.1.2 -oslotest==3.7.0 -yq>=2.7.2 -tox -pylibyaml~=0.1 -six>=1.15.0 -pyparsing>=2.1.0 -openstacksdk==0.36.5 +pifpaf==3.1.5 +oslotest==4.5.0 +yq==3.1.0 +tox<=4 +pylibyaml==0.1.0 +six==1.16.0 +pyparsing==2.4.7 +openstacksdk==0.59.0 +stestr==3.2.0 +requests==2.28.2 +urllib3==1.26.6 +chardet==4.0.0 +pyproject-api==1.5.0 +packaging>=23 \ No newline at end of file diff --git a/tools/gate/playbooks/osh-infra-deploy-docker.yaml b/tools/gate/playbooks/osh-infra-deploy-docker.yaml index 11638281..74d696b1 100644 --- a/tools/gate/playbooks/osh-infra-deploy-docker.yaml +++ b/tools/gate/playbooks/osh-infra-deploy-docker.yaml @@ -20,9 +20,9 @@ gather_facts: False become: yes roles: - - deploy-python + - ensure-python tags: - - deploy-python + - ensure-python - hosts: all vars_files: @@ -31,15 +31,20 @@ work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" gather_facts: True become: yes + roles: - - setup-firewall + - clear-firewall + - ensure-python + - ensure-pip + - ensure-docker + - ensure-tox - install-test-requirements - - deploy-python-pip - - deploy-docker - deploy-jq tags: - - setup-firewall + - clear-firewall + - ensure-python + - ensure-pip + - ensure-docker + - ensure-tox - install-test-requirements - - deploy-python-pip - - deploy-docker - deploy-jq diff --git a/tools/gate/playbooks/osh-infra-upgrade-host.yaml b/tools/gate/playbooks/osh-infra-upgrade-host.yaml index 3a2b79bb..b4a49c6a 100644 --- a/tools/gate/playbooks/osh-infra-upgrade-host.yaml +++ b/tools/gate/playbooks/osh-infra-upgrade-host.yaml @@ -20,9 +20,9 @@ gather_facts: False become: yes roles: - - deploy-python + - ensure-python tags: - - deploy-python + - ensure-python - hosts: all vars_files: diff --git a/tools/gate/playbooks/vars.yaml b/tools/gate/playbooks/vars.yaml index 7e45b7e1..f18a6cf6 100644 --- a/tools/gate/playbooks/vars.yaml +++ b/tools/gate/playbooks/vars.yaml @@ -12,4 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -barbican_stable_commit: 3ac3caa0138d44aa6031848d5b83802128a169b5 +# zuul_airship_deckhand_relative_path: ../airship-deckhand +# barbican_stable_commit: 3ac3caa0138d44aa6031848d5b83802128a169b5 diff --git a/tools/gate/roles/build-images/defaults/main.yaml b/tools/gate/roles/build-images/defaults/main.yaml index 7a7ad7a6..6447f48e 100644 --- a/tools/gate/roles/build-images/defaults/main.yaml +++ b/tools/gate/roles/build-images/defaults/main.yaml @@ -17,4 +17,4 @@ proxy: https: null noproxy: null -distro: ubuntu_bionic +distro: ubuntu_focal diff --git a/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml b/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml index 13e8ed00..388546d8 100644 --- a/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml +++ b/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml @@ -36,6 +36,7 @@ docker build \ --network host \ --force-rm \ + --tag deckhand \ --label zuul \ --file images/deckhand/Dockerfile.{{ distro }} \ {% if zuul_site_mirror_fqdn is defined and zuul_site_mirror_fqdn %} @@ -57,6 +58,7 @@ docker build \ --network host \ --force-rm \ + --tag deckhand \ --label zuul \ --file images/deckhand/Dockerfile.{{ distro }} \ --build-arg HTTP_PROXY="{{ proxy.http }}" \ @@ -81,4 +83,4 @@ register: airship_deckhand_image_id args: chdir: "{{ airship_deckhand_path.stdout }}" - become: yes + become: yes \ No newline at end of file diff --git a/tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml b/tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml index 765a4edd..e353c88e 100644 --- a/tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml +++ b/tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml @@ -15,7 +15,7 @@ - name: Deploy Barbican shell: | set -xe; - git reset --hard "{{ barbican_stable_commit | default('') }}" + # git reset --hard "{{ barbican_stable_commit | default('') }}" ./tools/deployment/component/barbican/barbican.sh args: chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" diff --git a/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml b/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml index 6ffec4e4..304894d9 100644 --- a/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml +++ b/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml @@ -20,6 +20,7 @@ docker run \ --rm \ --net=host \ + --dns 10.96.0.10 \ -v "{{ deckhand_conf_dir | default('') }}":/etc/deckhand "{{ airship_deckhand_image_id.stdout }}" alembic upgrade head # Allow migrations to complete. @@ -28,10 +29,13 @@ - name: Deploy Deckhand Container shell: |- + set -ex; + docker run \ --rm \ --net=host \ -p 9000:9000 \ + --dns 10.96.0.10 \ -v "{{ deckhand_conf_dir | default('') }}":/etc/deckhand "{{ airship_deckhand_image_id.stdout }}" server & # Give the server a chance to come up. Better to poll a health check. diff --git a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml index 230bcb73..3d95ee16 100644 --- a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml +++ b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml @@ -61,6 +61,6 @@ environment: DECKHAND_TEST_URL: "127.0.0.1:9000" DECKHAND_TEST_DIR: "{{ airship_deckhand_path.stdout }}/deckhand/tests/integration/gabbits" - BARBICAN_STABLE_COMMIT: "{{ barbican_stable_commit }}" + # BARBICAN_STABLE_COMMIT: "{{ barbican_stable_commit }}" register: result failed_when: "result.rc != 0" diff --git a/tools/gate/scripts/020-deploy-postgresql.sh b/tools/gate/scripts/020-deploy-postgresql.sh index 75076ddb..a902842a 100755 --- a/tools/gate/scripts/020-deploy-postgresql.sh +++ b/tools/gate/scripts/020-deploy-postgresql.sh @@ -23,7 +23,7 @@ POSTGRES_ID=$( -e POSTGRES_DB=deckhand \ -e POSTGRES_USER=deckhand \ -e POSTGRES_PASSWORD=password \ - postgres:9.5 + postgres:14.6 ) POSTGRES_IP=$( diff --git a/tools/helm_install.sh b/tools/helm_install.sh index 6e4c042c..14b08598 100755 --- a/tools/helm_install.sh +++ b/tools/helm_install.sh @@ -17,7 +17,7 @@ set -x HELM=$1 -HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v2.17.0-linux-amd64.tar.gz"} +HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.6.3-linux-amd64.tar.gz"} function install_helm_binary { diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh index 7fffed44..34a0aa9d 100755 --- a/tools/helm_tk.sh +++ b/tools/helm_tk.sh @@ -12,58 +12,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Script to setup helm-toolkit and helm dep up the deckhand chart -# -HELM=$1 -HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"} -HTK_PATH=${HTK_PATH:-""} + + +set -eux + +HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"} HTK_STABLE_COMMIT=${HTK_COMMIT:-"f4972121bcb41c8d74748917804d2b239ab757f9"} -BUILD_DIR=${BUILD_DIR:-$(mktemp -d)} -if [[ ! -z $(echo $http_proxy) ]] -then - export no_proxy=$no_proxy,127.0.0.1 -fi +TMP_DIR=$(mktemp -d) -set -x - -function helm_serve { - if [[ -d "$HOME/.helm" ]]; then - echo ".helm directory found" - else - ${HELM} init --client-only --skip-refresh - fi - if [[ -z $(curl -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; then - ${HELM} serve & > /dev/null - while [[ -z $(curl -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; do - sleep 1 - echo "Waiting for Helm Repository" - done - else - echo "Helm serve already running" - fi - - if ${HELM} repo list | grep -q "^stable" ; then - ${HELM} repo remove stable - fi - - ${HELM} repo add local http://localhost:8879/charts +{ + HTK_REPO_DIR=$TMP_DIR/htk + git clone "$HTK_REPO" "$HTK_REPO_DIR" + (cd "$HTK_REPO_DIR" && git reset --hard "${HTK_STABLE_COMMIT}") + cp -r "${HTK_REPO_DIR}/helm-toolkit" charts/deps/ } -mkdir -p "$BUILD_DIR" -pushd "$BUILD_DIR" -git clone $HTK_REPO || true -pushd openstack-helm-infra/$HTK_PATH -git reset --hard "${HTK_STABLE_COMMIT}" - -helm_serve -# OSH Makefile is bugged, so ensure helm is in the path -if [[ ${HELM} != "helm" ]] -then - export PATH=${PATH}:$(dirname ${HELM}) -fi - -make helm-toolkit -popd && popd -${HELM} dep up charts/deckhand +rm -rf "${TMP_DIR}" diff --git a/tools/integration-tests.sh b/tools/integration-tests.sh index 8dc63161..e2b7e755 100755 --- a/tools/integration-tests.sh +++ b/tools/integration-tests.sh @@ -47,7 +47,7 @@ function deploy_osh_keystone_barbican { fi cd ${OSH_PATH} - git reset --hard ${BARBICAN_STABLE_COMMIT} + # git reset --hard ${BARBICAN_STABLE_COMMIT} # Deploy required packages ./tools/deployment/common/install-packages.sh # Deploy Kubernetes diff --git a/tools/run_pifpaf.sh b/tools/run_pifpaf.sh index 358dd9e6..b9cfef20 100755 --- a/tools/run_pifpaf.sh +++ b/tools/run_pifpaf.sh @@ -14,7 +14,7 @@ trap cleanup EXIT # variables for debugging purposes. set -ex if [ -z $(which pg_config) ]; then - sudo apt-get install libpq-dev -y + sudo apt-get install libpq-dev postgresql -y fi eval `pifpaf run postgresql` diff --git a/tox.ini b/tox.ini index dca83033..34d51ccf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,14 @@ [tox] -minversion = 2.3.1 +minversion = 3.4 +# Flag indicating to perform the packaging operation or not. +# Set it to true when using tox for an application, instead of a library. skipsdist = True -envlist = py{37,36},py{37,36}-{postgresql},functional,cover,pep8,bandit,docs +envlist = py38,py38-{postgresql},functional,cover,pep8,bandit,docs [testenv] +# Install the current package in development mode with develop mode usedevelop = True -whitelist_externals = bash +allowlist_externals = bash find rm flake8 @@ -13,7 +16,21 @@ setenv = VIRTUAL_ENV={envdir} OS_TEST_PATH=./deckhand/tests/unit LANGUAGE=en_US LC_ALL=en_US.utf-8 -passenv = OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_TEST_TIMEOUT OS_TEST_LOCK_PATH OS_TEST_PATH http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY DECKHAND_IMAGE DECKHAND_TEST_URL DECKHAND_TEST_DIR +passenv = + OS_STDOUT_CAPTURE + OS_STDERR_CAPTURE + OS_TEST_TIMEOUT + OS_TEST_LOCK_PATH + OS_TEST_PATH + http_proxy + HTTP_PROXY + https_proxy + HTTPS_PROXY + no_proxy + NO_PROXY + DECKHAND_IMAGE + DECKHAND_TEST_URL + DECKHAND_TEST_DIR deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = @@ -25,18 +42,20 @@ basepython=python3 commands = {posargs} -[testenv:py36] -basepython = python3 +[testenv:py38] commands = {[testenv]commands} stestr run {posargs} stestr slowest -[testenv:py36-postgresql] -basepython = python3 +[testenv:py38-postgresql] commands = {[testenv]commands} - {toxinidir}/tools/run_pifpaf.sh '{posargs}' + bash {toxinidir}/tools/run_pifpaf.sh '{posargs}' +allowlist_externals = + bash + find + rm [testenv:functional] basepython=python3 @@ -45,7 +64,7 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = find . -type f -name "*.pyc" -delete - stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color '{posargs}' + stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color {posargs} [testenv:functional-dev] basepython=python3 @@ -106,7 +125,7 @@ commands = # [H904] Delay string interpolations at logging calls. enable-extensions = H106,H203,H204,H205,H210,H904 # [E731] Do not assign a lambda expression, use a def. This reduces readability in some cases. -ignore = E731,F405,H405,W504 +ignore = E731,F405,H405,W504,H306 exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,releasenotes,doc,alembic/versions [testenv:docs] @@ -114,7 +133,7 @@ basepython = python3 deps = -r{toxinidir}/doc/requirements.txt commands = - {toxinidir}/tools/build-docs.sh + bash {toxinidir}/tools/build-docs.sh [testenv:releasenotes] basepython = python3 @@ -122,5 +141,5 @@ deps = -r{toxinidir}/doc/requirements.txt commands = rm -rf releasenotes/build sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html -whitelist_externals = +allowlist_externals = rm