diff --git a/.dockerignore b/.dockerignore index 14a1bfc2..8d2a2e01 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,121 @@ -.tox -build +# Byte-compiled / optimized / DLL files +**/__pycache__/ +*.py[cod] +*$py.class + +# Unit tests +src/bin/*/tests/unit/ + +# C extensions +*.so + +# Distribution / packaging +.Python +**/env/ +**/build/ +**/develop-eggs/ +**/dist/ +**/downloads/ +**/eggs/ +**/.eggs/ +lib/ +**/lib64/ +parts/ +**/sdist/ +var/ +wheels/ +**/*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +**/htmlcov/ +cov/* +**/.tox/ +**/.coverage +**/.coverage.* +**/.pytest_cache/ +**/.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +doc/_build/ +doc/*/_static/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mypy +.mypy_cache/ + +# Generated bogus docs +ChangeLog +AUTHORS + +# build/lint artifacts +*.tgz +**/*.tgz +/charts/shipyard/charts +/charts/shipyard/requirements.lock +/charts/deps/*/ +.DS_Store + +# vscode +.vscode/ + +!src/lib/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index dc508da3..6ad6eac3 100644 --- a/.gitignore +++ b/.gitignore @@ -111,6 +111,7 @@ ENV/ # makefile build/lint artifacts /charts/deckhand/*.tgz /charts/deckhand/*.lock +/charts/deps/*/ # git Changelog AUTHORS diff --git a/.zuul.yaml b/.zuul.yaml index b4b8ea91..971dd6e4 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -14,52 +14,51 @@ templates: - openstack-cover-jobs - docs-on-readthedocs - - openstack-python36-jobs - - openstack-python3-train-jobs - - openstack-python3-ussuri-jobs + - openstack-python38-jobs vars: rtd_webhook_id: '38572' rtd_project_name: 'airship-deckhand' check: jobs: - - deckhand-tox-py36-postgresql - - deckhand-functional-uwsgi-py36 - - deckhand-functional-docker-py36-ubuntu_xenial - - deckhand-functional-docker-py36-ubuntu_bionic - - deckhand-functional-docker-py36-opensuse - - deckhand-integration-uwsgi-py36 - - deckhand-integration-docker-py36-ubuntu_xenial - - deckhand-integration-docker-py36-ubuntu_bionic - - deckhand-integration-docker-py36-opensuse + - deckhand-tox-py38-postgresql + # Remove in favor of deckhand-functional-docker-py38 it runs the + # same tests just in a container + # - deckhand-functional-uwsgi-py38 + - deckhand-functional-docker-py38-ubuntu_focal + # non-voting, failing, incompatable with updates + # - deckhand-functional-docker-py36-ubuntu_bionic + # non-voting, failing, incompatable with updates + # Remove in favor of deckhand-uwsgi-docker-py38 it runs the + # same tests just in a container + # - deckhand-integration-uwsgi-py38 + - deckhand-integration-docker-py38-ubuntu_focal + # non-voting, failing, incompatable with updates + # - deckhand-integration-docker-py36-ubuntu_bionic - deckhand-chart-build-gate - deckhand-chart-build-latest-htk - - deckhand-docker-build-gate-ubuntu_xenial - - deckhand-docker-build-gate-ubuntu_bionic - - deckhand-docker-build-gate-opensuse + - deckhand-docker-build-gate-ubuntu_focal + # non-voting, failing, incompatable with updates + # - deckhand-docker-build-gate-ubuntu_bionic + - openstack-tox-pep8 - deckhand-airskiff-deployment gate: jobs: - - deckhand-tox-py36-postgresql - - deckhand-functional-docker-py36-ubuntu_xenial + - deckhand-tox-py38-postgresql + - deckhand-functional-docker-py38-ubuntu_focal - deckhand-functional-docker-py36-ubuntu_bionic - - deckhand-functional-docker-py36-opensuse - - deckhand-integration-docker-py36-ubuntu_xenial + - deckhand-integration-docker-py38-ubuntu_focal - deckhand-integration-docker-py36-ubuntu_bionic - - deckhand-integration-docker-py36-opensuse - deckhand-chart-build-gate - - deckhand-docker-build-gate-ubuntu_xenial + - deckhand-docker-build-gate-ubuntu_focal - deckhand-docker-build-gate-ubuntu_bionic - - deckhand-docker-build-gate-opensuse - openstack-tox-pep8 post: jobs: - deckhand-upload-git-mirror - - deckhand-docker-publish-ubuntu_xenial + - deckhand-docker-publish-ubuntu_focal - deckhand-docker-publish-ubuntu_bionic - - deckhand-docker-publish-opensuse - - deckhand-docker-tag-ubuntu_xenial + - deckhand-docker-tag-ubuntu_focal - deckhand-docker-tag-ubuntu_bionic - - deckhand-docker-tag-opensuse - nodeset: name: deckhand-single-node @@ -73,22 +72,36 @@ - name: primary label: ubuntu-bionic +- nodeset: + name: deckhand-single-node-focal + nodes: + - name: primary + label: ubuntu-focal + +- nodeset: + name: deckhand-single-node-jammy + nodes: + - name: primary + label: ubuntu-jammy + - job: - name: deckhand-tox-py36-postgresql - parent: openstack-tox-py36 + name: deckhand-tox-py38-postgresql + parent: openstack-tox-py38 + nodeset: deckhand-single-node-focal pre-run: - tools/gate/playbooks/install-postgresql.yaml vars: - tox_envlist: py36-postgresql + tox_envlist: py38-postgresql - job: - name: deckhand-functional-uwsgi-py36 + name: deckhand-functional-uwsgi-py38 + voting: false description: | Run tox-based functional tests for the Airship Deckhand project using a minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf - for ephemeral PostgreSQL DB, under cPython version 3.6. + for ephemeral PostgreSQL DB, under cPython version 3.8. run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal vars: tox_envlist: functional-dev irrelevant-files: &irrelevant-files @@ -104,6 +117,7 @@ description: | Base job for running deckhand functional tests. Runs tests against Docker image generated from source code. + nodeset: deckhand-single-node-focal roles: - zuul: openstack/openstack-helm-infra timeout: 3600 @@ -115,22 +129,22 @@ - openstack/openstack-helm-infra - job: - name: deckhand-functional-docker-py36-ubuntu_xenial - voting: false + name: deckhand-functional-docker-py38-ubuntu_focal description: | Run tox-based functional tests for the Airship Deckhand project under - cPython version 3.6. Uses tox with the ``functional-py36`` environment. - Ubuntu (xenial) image is built and used. + cPython version 3.8. Uses tox with the ``functional-py38`` environment. + Ubuntu (focal) image is built and used. parent: deckhand-functional-docker-base - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal vars: tox_envlist: functional disable_keystone: true - distro: ubuntu_xenial + distro: ubuntu_focal irrelevant-files: *irrelevant-files - job: name: deckhand-functional-docker-py36-ubuntu_bionic + voting: false description: | Run tox-based functional tests for the Airship Deckhand project under cPython version 3.6. Uses tox with the ``functional-py36`` environment. @@ -144,29 +158,15 @@ irrelevant-files: *irrelevant-files - job: - name: deckhand-functional-docker-py36-opensuse + name: deckhand-integration-uwsgi-py38 voting: false - description: | - Run tox-based functional tests for the Airship Deckhand project under - cPython version 3.6. Uses tox with the ``functional-py36`` environment. - Opensuse image is built and used. - parent: deckhand-functional-docker-base - nodeset: deckhand-single-node - vars: - tox_envlist: functional - disable_keystone: true - distro: opensuse_15 - irrelevant-files: *irrelevant-files - -- job: - name: deckhand-integration-uwsgi-py36 description: | Run tox-based integration tests for the Airship Deckhand project using a minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf - for ephemeral PostgreSQL DB, under cPython version 3.6. + for ephemeral PostgreSQL DB, under cPython version 3.8. timeout: 3600 run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal irrelevant-files: *irrelevant-files vars: disable_keystone: true @@ -177,7 +177,7 @@ Build charts using pinned Helm toolkit. timeout: 900 run: tools/gate/playbooks/build-charts.yaml - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal - job: name: deckhand-chart-build-latest-htk @@ -185,7 +185,7 @@ Build charts using latest Helm toolkit. timeout: 900 run: tools/gate/playbooks/build-charts.yaml - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal vars: HTK_COMMIT: master @@ -194,6 +194,7 @@ description: | Base job for running deckhand integration tests. Runs tests against Docker image generated from source code. + nodeset: deckhand-single-node-focal timeout: 3600 roles: - zuul: openstack/openstack-helm-infra @@ -203,7 +204,8 @@ pre-run: - tools/gate/playbooks/osh-infra-upgrade-host.yaml - tools/gate/playbooks/osh-infra-deploy-docker.yaml - run: tools/gate/playbooks/run-integration-tests-docker.yaml + run: + - tools/gate/playbooks/run-integration-tests-docker.yaml post-run: tools/gate/playbooks/osh-infra-collect-logs.yaml required-projects: - openstack/openstack-helm @@ -214,16 +216,15 @@ - ^releasenotes/.*$ - job: - name: deckhand-integration-docker-py36-ubuntu_xenial - voting: false + name: deckhand-integration-docker-py38-ubuntu_focal description: | Run tox-based integration tests for the Airship Deckhand project under - cPython version 3.6. Builds ubuntu (xenial) deckhand image. + cPython version 3.8. Builds ubuntu (focal) deckhand image. parent: deckhand-integration-docker-base nodeset: openstack-helm-single-node vars: disable_keystone: false - distro: ubuntu_xenial + distro: ubuntu_focal - job: name: deckhand-integration-docker-py36-ubuntu_bionic @@ -259,20 +260,7 @@ - ^releasenotes/.*$ - job: - name: deckhand-integration-docker-py36-opensuse - voting: false - description: | - Run tox-based integration tests for the Airship Deckhand project under - cPython version 3.6. Builds opensuse deckhand image. - parent: deckhand-integration-docker-base - nodeset: openstack-helm-single-node - vars: - disable_keystone: false - distro: opensuse_15 - -- job: - name: deckhand-docker-build-gate-ubuntu_xenial - voting: false + name: deckhand-docker-build-gate-ubuntu_focal timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml nodeset: deckhand-single-node @@ -285,13 +273,14 @@ - ^setup.cfg$ vars: publish: false - distro: ubuntu_xenial + distro: ubuntu_focal tags: dynamic: patch_set: true - job: name: deckhand-docker-build-gate-ubuntu_bionic + voting: false timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml nodeset: deckhand-single-node @@ -304,35 +293,20 @@ patch_set: true - job: - name: deckhand-docker-build-gate-opensuse - voting: false - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node - irrelevant-files: *non-code-files-template - vars: - publish: false - distro: opensuse_15 - tags: - dynamic: - patch_set: true - -- job: - name: deckhand-docker-publish-ubuntu_xenial - voting: false + name: deckhand-docker-publish-ubuntu_focal description: | Runs on every merge, unless files in a dictionary below are changed. Builds and publishes container ubuntu images on quay.io with a set of tags listed in vars section. Waits in Zuul queue for a node (VM) assignment. timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal secrets: - airship_deckhand_quay_creds irrelevant-files: *non-code-files-template vars: publish: true - distro: ubuntu_xenial + distro: ubuntu_focal tags: dynamic: branch: true @@ -342,6 +316,7 @@ - job: name: deckhand-docker-publish-ubuntu_bionic + voting: false description: | Runs on every merge, unless files in a dictionary below are changed. Builds and publishes container ubuntu images on quay.io with a set of tags @@ -363,31 +338,7 @@ - latest - job: - name: deckhand-docker-publish-opensuse - voting: false - description: | - Runs on every merge, unless files in a dictionary below are changed. - Builds and publishes container opensuse images on quay.io with a set of tags - listed in vars section. Waits in Zuul queue for a node (VM) assignment. - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node - secrets: - - airship_deckhand_quay_creds - irrelevant-files: *non-code-files-template - vars: - publish: true - distro: opensuse_15 - tags: - dynamic: - branch: true - commit: true - static: - - latest - -- job: - name: deckhand-docker-tag-ubuntu_xenial - voting: false + name: deckhand-docker-tag-ubuntu_focal description: | Runs on every merge when files in a dictionalry below are changed, and adds git commit id tag onto the ubuntu container image published on quay.io, @@ -400,10 +351,11 @@ secrets: - airship_deckhand_quay_creds vars: - distro: ubuntu_xenial + distro: ubuntu_focal - job: name: deckhand-docker-tag-ubuntu_bionic + voting: false description: | Runs on every merge when files in a dictionalry below are changed, and adds git commit id tag onto the ubuntu container image published on quay.io, @@ -418,27 +370,6 @@ vars: distro: ubuntu_bionic -- job: - name: deckhand-docker-tag-opensuse - voting: false - description: | - Runs on every merge when files in a dictionalry below are changed, and - adds git commit id tag onto the opensuse container image published on quay.io, - which has `latest` tag set. Does not wait in queue for a node (VM) - assignment, runs almost immediately. - timeout: 1800 - run: tools/gate/playbooks/docker-image-tag.yaml - nodeset: - nodes: [] - secrets: - - airship_deckhand_quay_creds - vars: - distro: opensuse_15 -# file pattern here must be exactly the same as in -# deckhand-docker-publish job above, -# job will be executed on merge only when any of this files get changed - files: *non-code-files-template - - secret: name: airship_deckhand_quay_creds data: diff --git a/Makefile b/Makefile index 498bb281..c3b7f280 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ USE_PROXY ?= false PUSH_IMAGE ?= false # use this variable for image labels added in internal build process LABEL ?= org.airshipit.build=community -DISTRO ?= ubuntu_bionic +DISTRO ?= ubuntu_focal COMMIT ?= $(shell git rev-parse HEAD) IMAGE := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} @@ -37,13 +37,13 @@ images: build_deckhand # Create tgz of the chart .PHONY: charts -charts: helm-init +charts: helm-toolkit $(HELM) dep up charts/deckhand $(HELM) package charts/deckhand # Initialize local helm config -.PHONY: helm-init -helm-init: helm-install +.PHONY: helm-toolkit +helm-toolkit: helm-install tools/helm_tk.sh $(HELM) # Install helm binary diff --git a/charts/deckhand/Chart.yaml b/charts/deckhand/Chart.yaml index 87223af6..298bc7ed 100644 --- a/charts/deckhand/Chart.yaml +++ b/charts/deckhand/Chart.yaml @@ -15,7 +15,8 @@ apiVersion: v1 description: A Helm chart for Deckhand name: deckhand -version: 0.1.0 +version: 0.2.0 +appVersion: 1.0 keywords: - deckhand home: https://github.com/openstack/airship-deckhand diff --git a/charts/deckhand/requirements.yaml b/charts/deckhand/requirements.yaml index c81c2135..f0eabffd 100644 --- a/charts/deckhand/requirements.yaml +++ b/charts/deckhand/requirements.yaml @@ -14,5 +14,5 @@ dependencies: - name: helm-toolkit - repository: http://localhost:8879/charts + repository: file://../deps/helm-toolkit version: ">= 0.1.0" diff --git a/charts/deps/.gitkeep b/charts/deps/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/deckhand/barbican/driver.py b/deckhand/barbican/driver.py index 83dc9291..b269ef73 100644 --- a/deckhand/barbican/driver.py +++ b/deckhand/barbican/driver.py @@ -163,7 +163,7 @@ class BarbicanDriver(object): raise errors.BarbicanServerException(details=str(e)) payload = secret.payload - if secret.secret_type == 'opaque': + if secret.secret_type == 'opaque': # nosec LOG.debug('Base64-decoding original payload ' 'for document [%s, %s] %s.', *src_doc.meta) secret = self._base64_decode_payload(payload) @@ -182,7 +182,7 @@ class BarbicanDriver(object): except (barbicanclient.exceptions.HTTPAuthError, barbicanclient.exceptions.HTTPServerError) as e: LOG.exception(str(e)) - raise errors.BarbicanException(details=str(e)) + raise errors.BarbicanClientException(details=str(e)) except barbicanclient.exceptions.HTTPClientError as e: if e.status_code == 404: LOG.warning('Could not delete secret %s because it was not ' diff --git a/deckhand/common/document.py b/deckhand/common/document.py index 6d7f9155..86072a4c 100644 --- a/deckhand/common/document.py +++ b/deckhand/common/document.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections +from collections.abc import Iterable import re import hashlib @@ -167,7 +167,7 @@ class DocumentDict(dict): :param documents: Documents to wrap in this class. :type documents: iterable """ - if not isinstance(documents, collections.Iterable): + if not isinstance(documents, Iterable): documents = [documents] return [DocumentDict(d) for d in documents] diff --git a/deckhand/common/utils.py b/deckhand/common/utils.py index 2516e6ed..9de3747c 100644 --- a/deckhand/common/utils.py +++ b/deckhand/common/utils.py @@ -49,7 +49,7 @@ def to_camel_case(s): def to_snake_case(name): """Convert string to snake case.""" - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', str(name)) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() diff --git a/deckhand/control/base.py b/deckhand/control/base.py index 897db2ed..b15a6908 100644 --- a/deckhand/control/base.py +++ b/deckhand/control/base.py @@ -55,7 +55,7 @@ class BaseResource(object): :param allow_empty: Whether the request body can be empty. :returns: List of dicts if ``expect_list`` is True or else a dict. """ - raw_data = req.stream.read(req.content_length or 0) + raw_data = req.bounded_stream.read() if not allow_empty and not raw_data: error_msg = ("The request body must not be empty.") diff --git a/deckhand/control/buckets.py b/deckhand/control/buckets.py index dc531966..2d58bb62 100644 --- a/deckhand/control/buckets.py +++ b/deckhand/control/buckets.py @@ -64,7 +64,7 @@ class BucketsResource(api_base.BaseResource): created_documents = self._create_revision_documents( bucket_name, documents) - resp.body = self.view_builder.list(created_documents) + resp.text = self.view_builder.list(created_documents) resp.status = falcon.HTTP_200 def _encrypt_secret_documents(self, documents): diff --git a/deckhand/control/middleware.py b/deckhand/control/middleware.py index 7acd8c10..c5cb3127 100644 --- a/deckhand/control/middleware.py +++ b/deckhand/control/middleware.py @@ -161,7 +161,7 @@ class YAMLTranslator(HookableMiddlewareMixin, object): ) raise falcon.HTTPUnsupportedMediaType(description=message) - def process_response(self, req, resp, resource): + def process_response(self, req, resp, resource, req_succeeded): """Converts responses to ``application/x-yaml`` content type.""" if resp.status != '204 No Content': resp.set_header('Content-Type', 'application/x-yaml') diff --git a/deckhand/engine/revision_diff.py b/deckhand/engine/revision_diff.py index e218630e..f8373c1c 100644 --- a/deckhand/engine/revision_diff.py +++ b/deckhand/engine/revision_diff.py @@ -248,7 +248,8 @@ def _diff_buckets(b1, b2): # serializing to json then deserializing # to dict. data_changed = jsonpickle.decode( - DeepDiff(d['data'], b2_tmp[k]['data']).json) + DeepDiff(d['data'], b2_tmp[k]['data']).to_json()) + # deepdiff doesn't provide custom exceptions; # have to use Exception. except Exception as ex: @@ -256,7 +257,7 @@ def _diff_buckets(b1, b2): try: metadata_changed = jsonpickle.decode( DeepDiff(d['metadata'], - b2_tmp[k]['metadata']).json) + b2_tmp[k]['metadata']).to_json()) except Exception as ex: raise errors.DeepDiffException(details=str(ex)) diff --git a/deckhand/engine/utils.py b/deckhand/engine/utils.py index 32e85eb6..11562cf8 100644 --- a/deckhand/engine/utils.py +++ b/deckhand/engine/utils.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping from deckhand.common import utils @@ -32,7 +35,7 @@ def deep_merge(dct, merge_dct): """ for k, v in merge_dct.items(): if (k in dct and isinstance(dct[k], dict) and - isinstance(merge_dct[k], collections.Mapping)): + isinstance(merge_dct[k], Mapping)): deep_merge(dct[k], merge_dct[k]) else: dct[k] = merge_dct[k] diff --git a/deckhand/errors.py b/deckhand/errors.py index 9d317247..a9e0bbdd 100644 --- a/deckhand/errors.py +++ b/deckhand/errors.py @@ -102,10 +102,10 @@ def format_error_resp(req, } resp.status = status_code - resp.body = yaml.safe_dump(error_response) + resp.text = yaml.safe_dump(error_response) -def default_exception_handler(ex, req, resp, params): +def default_exception_handler(req, resp, ex, params): """Catch-all exception handler for standardized output. If this is a standard falcon HTTPError, rethrow it for handling by diff --git a/deckhand/service.py b/deckhand/service.py index e1137adf..a514c6f9 100644 --- a/deckhand/service.py +++ b/deckhand/service.py @@ -17,7 +17,7 @@ import os import falcon from oslo_config import cfg from oslo_log import log -from werkzeug.contrib.profiler import ProfilerMiddleware +from werkzeug.middleware.profiler import ProfilerMiddleware from deckhand.control import base from deckhand.control import buckets @@ -88,7 +88,7 @@ def deckhand_app_factory(global_config, **local_config): middleware.ContextMiddleware(), middleware.LoggingMiddleware()] - app = falcon.API(request_type=base.DeckhandRequest, + app = falcon.App(request_type=base.DeckhandRequest, middleware=middleware_list) if CONF.profiler: LOG.warning("Profiler ENABLED. Expect significant " diff --git a/deckhand/tests/functional/gabbits/revision-deepdiff/revision-deepdiff-success.yaml b/deckhand/tests/functional/gabbits/revision-deepdiff/revision-deepdiff-success.yaml index 18e3ab25..9d8497fe 100644 --- a/deckhand/tests/functional/gabbits/revision-deepdiff/revision-deepdiff-success.yaml +++ b/deckhand/tests/functional/gabbits/revision-deepdiff/revision-deepdiff-success.yaml @@ -241,7 +241,8 @@ tests: $.[0].'bucket_a diff'.document_changed.details: ('example/Kind/v1', 'doc-a'): data_changed: - dictionary_item_added: !!set {"root['foo']"} + dictionary_item_added: + ["root['foo']"] metadata_changed: {} - name: removing_key_in_doc-a @@ -273,7 +274,8 @@ tests: $.[0].'bucket_a diff'.document_changed.details: ('example/Kind/v1', 'doc-a'): data_changed: - dictionary_item_removed: !!set {"root['foo']"} + dictionary_item_removed: + ["root['foo']"] metadata_changed: {} - name: verify_invalid_input diff --git a/deckhand/tests/unit/barbican/test_cache.py b/deckhand/tests/unit/barbican/test_cache.py index 1321aaea..0b32483f 100644 --- a/deckhand/tests/unit/barbican/test_cache.py +++ b/deckhand/tests/unit/barbican/test_cache.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import testtools from deckhand.barbican import cache diff --git a/deckhand/tests/unit/base.py b/deckhand/tests/unit/base.py index 085bb6cf..9b637614 100644 --- a/deckhand/tests/unit/base.py +++ b/deckhand/tests/unit/base.py @@ -17,7 +17,7 @@ from __future__ import absolute_import import os import fixtures -import mock +from unittest import mock from oslo_config import cfg from oslo_log import log as logging import testtools @@ -26,7 +26,7 @@ from deckhand.conf import config # noqa: Calls register_opts(CONF) from deckhand.db.sqlalchemy import api as db_api from deckhand.engine import cache from deckhand.tests import test_utils -from deckhand.tests.unit import fixtures as dh_fixtures +from deckhand.tests.unit import dh_fixtures CONF = cfg.CONF logging.register_options(CONF) diff --git a/deckhand/tests/unit/common/test_utils.py b/deckhand/tests/unit/common/test_utils.py index 238745ac..f31303de 100644 --- a/deckhand/tests/unit/common/test_utils.py +++ b/deckhand/tests/unit/common/test_utils.py @@ -14,7 +14,7 @@ import hashlib import jsonpath_ng -import mock +from unittest import mock from oslo_serialization import jsonutils as json from testtools.matchers import Equals diff --git a/deckhand/tests/unit/control/base.py b/deckhand/tests/unit/control/base.py index 34dc7448..822d5425 100644 --- a/deckhand/tests/unit/control/base.py +++ b/deckhand/tests/unit/control/base.py @@ -19,7 +19,7 @@ from falcon import testing as falcon_testing from deckhand import service from deckhand.tests.unit import base as test_base -from deckhand.tests.unit import fixtures +from deckhand.tests.unit import dh_fixtures class BaseControllerTest(test_base.DeckhandWithDBTestCase, @@ -30,10 +30,10 @@ class BaseControllerTest(test_base.DeckhandWithDBTestCase, super(BaseControllerTest, self).setUp() self.app = falcon_testing.TestClient( service.deckhand_app_factory(None)) - self.policy = self.useFixture(fixtures.RealPolicyFixture()) + self.policy = self.useFixture(dh_fixtures.RealPolicyFixture()) # NOTE: development_mode allows these unit tests to get around # Keystone authentication. - self.useFixture(fixtures.ConfPatcher(development_mode=True)) + self.useFixture(dh_fixtures.ConfPatcher(development_mode=True)) def _read_data(self, file_name): # Reads data from a file in the resources directory diff --git a/deckhand/tests/unit/control/test_api_initialization.py b/deckhand/tests/unit/control/test_api_initialization.py index cddfccda..b5bd875f 100644 --- a/deckhand/tests/unit/control/test_api_initialization.py +++ b/deckhand/tests/unit/control/test_api_initialization.py @@ -15,7 +15,7 @@ import inspect import os -import mock +from unittest import mock from deckhand.common import utils from deckhand.control import api @@ -72,7 +72,7 @@ class TestApi(test_base.DeckhandTestCase): @mock.patch('deckhand.service.falcon', autospec=True) def test_init_application(self, mock_falcon, mock_logging, mock_db_api, _): - mock_falcon_api = mock_falcon.API.return_value + mock_falcon_api = mock_falcon.App.return_value self.override_config( 'connection', mock.sentinel.db_connection, group='database') diff --git a/deckhand/tests/unit/control/test_base_controller.py b/deckhand/tests/unit/control/test_base_controller.py index 6f57dc6a..4c58bdea 100644 --- a/deckhand/tests/unit/control/test_base_controller.py +++ b/deckhand/tests/unit/control/test_base_controller.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock from deckhand.control import base as api_base from deckhand.tests.unit.control import base as test_base diff --git a/deckhand/tests/unit/control/test_buckets_controller.py b/deckhand/tests/unit/control/test_buckets_controller.py index 27a234b4..581e1b9a 100644 --- a/deckhand/tests/unit/control/test_buckets_controller.py +++ b/deckhand/tests/unit/control/test_buckets_controller.py @@ -14,7 +14,7 @@ import yaml -import mock +from unittest import mock from oslo_config import cfg from deckhand.engine import secrets_manager diff --git a/deckhand/tests/unit/control/test_errors.py b/deckhand/tests/unit/control/test_errors.py index 538973a0..b06e18e5 100644 --- a/deckhand/tests/unit/control/test_errors.py +++ b/deckhand/tests/unit/control/test_errors.py @@ -16,7 +16,7 @@ import os import yaml import falcon -import mock +from unittest import mock from deckhand.common import document as document_wrapper from deckhand import policy @@ -58,7 +58,7 @@ class TestErrorFormatting(test_base.BaseControllerTest): 'message': 'Unhandled Exception raised: test error', 'metadata': {} } - body = yaml.load(resp.text) + body = yaml.safe_load(resp.text) self.assertEqual(500, resp.status_code) self.assertEqual(expected, body) diff --git a/deckhand/tests/unit/control/test_middleware.py b/deckhand/tests/unit/control/test_middleware.py index 55fa862d..dea3f13f 100644 --- a/deckhand/tests/unit/control/test_middleware.py +++ b/deckhand/tests/unit/control/test_middleware.py @@ -14,7 +14,7 @@ import yaml -import mock +from unittest import mock from deckhand import factories from deckhand.tests.unit.control import base as test_base @@ -118,16 +118,17 @@ class TestYAMLTranslatorNegative(test_base.BaseControllerTest): 'errorType': 'HTTPMissingHeader', 'messageList': [{ 'error': True, - 'message': "The Content-Type header is required." + 'message': "The \"Content-Type\" header is required." }] }, 'kind': 'Status', - 'message': "The Content-Type header is required.", + 'message': "The \"Content-Type\" header is required.", 'metadata': {}, 'reason': 'Unspecified', 'retry': False, 'status': 'Failure' } + self.assertEqual(expected, yaml.safe_load(resp.content)) def test_request_with_invalid_content_type_raises_exception(self): diff --git a/deckhand/tests/unit/control/test_rendered_documents_controller.py b/deckhand/tests/unit/control/test_rendered_documents_controller.py index 9a022c21..1ab75a1b 100644 --- a/deckhand/tests/unit/control/test_rendered_documents_controller.py +++ b/deckhand/tests/unit/control/test_rendered_documents_controller.py @@ -15,7 +15,7 @@ import re import yaml -import mock +from unittest import mock from deckhand.common.document import DocumentDict as dd from deckhand.control import revision_documents diff --git a/deckhand/tests/unit/control/test_revision_documents_controller.py b/deckhand/tests/unit/control/test_revision_documents_controller.py index f8d0343d..c1abcc15 100644 --- a/deckhand/tests/unit/control/test_revision_documents_controller.py +++ b/deckhand/tests/unit/control/test_revision_documents_controller.py @@ -14,7 +14,7 @@ import yaml -import mock +from unittest import mock from deckhand.common.document import DocumentDict as document_dict from deckhand.engine import secrets_manager diff --git a/deckhand/tests/unit/control/test_revisions_controller.py b/deckhand/tests/unit/control/test_revisions_controller.py index cc38c45f..f191e6e5 100644 --- a/deckhand/tests/unit/control/test_revisions_controller.py +++ b/deckhand/tests/unit/control/test_revisions_controller.py @@ -14,7 +14,7 @@ import os -import mock +from unittest import mock import yaml from deckhand.engine import secrets_manager diff --git a/deckhand/tests/unit/control/test_revisions_rollback_controller.py b/deckhand/tests/unit/control/test_revisions_rollback_controller.py index 0ada8e11..49d43852 100644 --- a/deckhand/tests/unit/control/test_revisions_rollback_controller.py +++ b/deckhand/tests/unit/control/test_revisions_rollback_controller.py @@ -14,7 +14,7 @@ import yaml -import mock +from unittest import mock from deckhand.engine import secrets_manager from deckhand import factories diff --git a/deckhand/tests/unit/db/test_documents.py b/deckhand/tests/unit/db/test_documents.py index 246c4950..a05ef3c9 100644 --- a/deckhand/tests/unit/db/test_documents.py +++ b/deckhand/tests/unit/db/test_documents.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import testtools from deckhand.db.sqlalchemy import api as db_api diff --git a/deckhand/tests/unit/fixtures.py b/deckhand/tests/unit/dh_fixtures.py similarity index 99% rename from deckhand/tests/unit/fixtures.py rename to deckhand/tests/unit/dh_fixtures.py index 45e162bd..e8ee7ff2 100644 --- a/deckhand/tests/unit/fixtures.py +++ b/deckhand/tests/unit/dh_fixtures.py @@ -22,7 +22,7 @@ import os import yaml import fixtures -import mock +from unittest import mock from oslo_config import cfg from oslo_policy import opts as policy_opts from oslo_policy import policy as oslo_policy @@ -41,7 +41,6 @@ class ConfPatcher(fixtures.Fixture): it's teardown. """ - def __init__(self, **kwargs): """Constructor diff --git a/deckhand/tests/unit/engine/test_document_layering.py b/deckhand/tests/unit/engine/test_document_layering.py index 1fbf1fbf..464f7dac 100644 --- a/deckhand/tests/unit/engine/test_document_layering.py +++ b/deckhand/tests/unit/engine/test_document_layering.py @@ -14,7 +14,7 @@ import yaml -import mock +from unittest import mock from deckhand.engine import layering from deckhand.engine import secrets_manager diff --git a/deckhand/tests/unit/engine/test_document_layering_and_substitution.py b/deckhand/tests/unit/engine/test_document_layering_and_substitution.py index 5ccf58b7..3ccf88ea 100644 --- a/deckhand/tests/unit/engine/test_document_layering_and_substitution.py +++ b/deckhand/tests/unit/engine/test_document_layering_and_substitution.py @@ -14,7 +14,7 @@ import itertools -import mock +from unittest import mock import yaml from deckhand import factories diff --git a/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py b/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py index 2215c015..5c819ff0 100644 --- a/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py +++ b/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock from deckhand.engine import layering from deckhand import errors diff --git a/deckhand/tests/unit/engine/test_document_layering_negative.py b/deckhand/tests/unit/engine/test_document_layering_negative.py index 85a7f42e..04f8ba21 100644 --- a/deckhand/tests/unit/engine/test_document_layering_negative.py +++ b/deckhand/tests/unit/engine/test_document_layering_negative.py @@ -14,7 +14,7 @@ import copy -import mock +from unittest import mock from deckhand.engine import layering from deckhand import errors diff --git a/deckhand/tests/unit/engine/test_document_validation.py b/deckhand/tests/unit/engine/test_document_validation.py index 180ecd07..9a0a2c51 100644 --- a/deckhand/tests/unit/engine/test_document_validation.py +++ b/deckhand/tests/unit/engine/test_document_validation.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock from deckhand.common import utils from deckhand.engine import document_validation diff --git a/deckhand/tests/unit/engine/test_document_validation_negative.py b/deckhand/tests/unit/engine/test_document_validation_negative.py index 82c430ef..37b774d2 100644 --- a/deckhand/tests/unit/engine/test_document_validation_negative.py +++ b/deckhand/tests/unit/engine/test_document_validation_negative.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock from deckhand.engine import document_validation from deckhand import errors diff --git a/deckhand/tests/unit/engine/test_secrets_manager.py b/deckhand/tests/unit/engine/test_secrets_manager.py index 4a3bdf0e..c521e204 100644 --- a/deckhand/tests/unit/engine/test_secrets_manager.py +++ b/deckhand/tests/unit/engine/test_secrets_manager.py @@ -15,7 +15,7 @@ import copy import yaml -import mock +from unittest import mock from oslo_serialization import base64 from oslo_utils import uuidutils import testtools diff --git a/deckhand/tests/unit/test_policy.py b/deckhand/tests/unit/test_policy.py index cccfcd6c..198736d5 100644 --- a/deckhand/tests/unit/test_policy.py +++ b/deckhand/tests/unit/test_policy.py @@ -13,13 +13,14 @@ # limitations under the License. import falcon -import mock +from falcon import testing as falcon_testing +from unittest import mock from oslo_policy import policy as common_policy from deckhand.control import base as api_base import deckhand.policy from deckhand.tests.unit import base as test_base -from deckhand.tests.unit import fixtures +from deckhand.tests.unit import dh_fixtures class PolicyBaseTestCase(test_base.DeckhandTestCase): @@ -34,7 +35,7 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase): "deckhand:list_cleartext_documents": [['rule:admin_api']] } - self.useFixture(fixtures.RealPolicyFixture(False)) + self.useFixture(dh_fixtures.RealPolicyFixture(False)) self._set_rules() def _set_rules(self): @@ -51,9 +52,23 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase): noop(*api_args) def _get_args(self): + env = falcon_testing.create_environ( + path='/', + query_string='', + scheme='http', + host='falconframework.org', + port=None, + headers={'Content-Type': 'application/x-yaml'}, + app='', + body='', + method='POST', + wsgierrors=None, + file_wrapper=None) + + falcon_req = api_base.DeckhandRequest(env) + # Returns the first two arguments that would be passed to any falcon # on_{HTTP_VERB} method: (self (which is mocked), falcon Request obj). - falcon_req = api_base.DeckhandRequest(mock.MagicMock()) return (mock.Mock(), falcon_req) diff --git a/doc/requirements.txt b/doc/requirements.txt index ba09cc72..916fb8ad 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -4,6 +4,7 @@ sphinx>=1.6.2 # BSD sphinx_rtd_theme reno>=2.5.0 # Apache-2.0 +pylibyaml==0.1.0 plantuml sphinxcontrib-apidoc>=0.2.0 # BSD sphinxcontrib-plantuml diff --git a/images/deckhand/Dockerfile.ubuntu_xenial b/images/deckhand/Dockerfile.ubuntu_focal similarity index 88% rename from images/deckhand/Dockerfile.ubuntu_xenial rename to images/deckhand/Dockerfile.ubuntu_focal index 75341d6a..2fb34cdb 100644 --- a/images/deckhand/Dockerfile.ubuntu_xenial +++ b/images/deckhand/Dockerfile.ubuntu_focal @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG FROM=ubuntu:16.04 +ARG FROM=ubuntu:20.04 FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' @@ -51,7 +51,7 @@ RUN set -x && \ python3-pip \ python3-setuptools \ --no-install-recommends \ - && python3 -m pip install -U 'pip<21.0' \ + && python3 -m pip install -U pip \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ @@ -88,8 +88,16 @@ RUN chown -R deckhand: /home/deckhand \ # Set work directory and install dependencies WORKDIR /home/deckhand -RUN pip3 install -r requirements.txt -RUN python3 setup.py install +RUN pip3 install --no-cache-dir -r requirements.txt + +# Setting deckhand version for BPR +ENV PBR_VERSION 1.0 + + +COPY . /opt/deckhand +RUN pip3 install -e /opt/deckhand \ + && echo "/opt/deckhand" \ + > /usr/local/lib/python3.8/dist-packages/deckhand.pth # Set user to deckhand USER deckhand diff --git a/requirements.txt b/requirements.txt index 35f2b0dd..582b8c26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,51 +2,54 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=3.0.1,<3.1.0 - -alembic==1.0.1 -amqp<2.7,>=2.6.0 -beaker==1.10.0 -cryptography>=2.7 -deepdiff==3.3.0 -falcon==1.4.1 -jsonpath-ng==1.4.3 -jsonschema>=3.0.1,<4 -keystoneauth1>=3.18.0 -keystonemiddleware==5.3.0 -kombu<4.7,>=4.6.10 -networkx==2.2 -oslo.cache==1.38.1 -oslo.concurrency==3.28.1 -oslo.config==7.0.0 -oslo.context>=2.21.0 -oslo.messaging==9.1.1 -oslo.db==4.41.1 -oslo.log==3.45.2 -oslo.middleware==3.36.0 -oslo.policy==1.40.1 -oslo.serialization==2.29.2 -oslo.utils==3.42.1 -pbr==5.4.5 -PasteDeploy==1.5.2 -Paste==3.0.1 -psycopg2-binary==2.8.4 -pylibyaml~=0.1 -pyyaml~=5.1 -python-dateutil>=2.8.1 +hacking==3.0.1 +SQLAlchemy==1.4.23 +alembic==1.7.1 +amqp==5.0.8 +Beaker==1.12.0 +cryptography==3.4.8 +deepdiff==5.8.1 +falcon==3.1.1 +jsonpath-ng==1.5.3 +jsonschema==3.2.0 +keystoneauth1==5.1.1 +keystonemiddleware==10.2.0 +kombu==5.1.0 +networkx==2.6.2 +oslo.cache==2.8.2 +oslo.concurrency==4.4.1 +oslo.config==8.7.1 +oslo.context==5.0.0 +oslo.messaging==12.9.4 +oslo.db==11.0.0 +oslo.log==4.6.0 +oslo.middleware==4.4.0 +oslo.policy==4.0.0 +oslo.serialization==4.2.0 +oslo.utils==4.10.2 +pbr==5.6.0 +PasteDeploy==3.0.1 +Paste==3.5.0 +psycopg2-binary==2.9.5 +pylibyaml==0.1.0 +PyYAML==5.4.1 +python-dateutil==2.8.2 # TODO(alanmeadows) # this must match the container service # likely this should be imported from a # container sidecar long-term -python-barbicanclient==4.7.0 +python-barbicanclient==5.2.0 python-keystoneclient==3.22.0 python-memcached==1.59 -Routes==2.4.1 -six>=1.15.0 -stevedore>=1.30.0 -urllib3==1.25.9 -uwsgi~=2.0.19.1 +Routes==2.5.1 +six==1.16.0 +sphinx-rtd-theme==1.1.1 +stestr==3.2.0 +stevedore==4.1.1 +urllib3==1.26.6 +uWSGI==2.0.21 # To support profiling in non-prod -Werkzeug==0.16.1 +Werkzeug==2.0.1 +jsonpickle \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 7347ea3e..4a9d9ea6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,11 @@ [metadata] name = Deckhand +version = 1.0 summary = Storage service for YAML-based configuration documents, which are managed through version control and automatically validated. -description-file = README.rst +description_file = README.rst author = The Airship Authors -author-email = airship-discuss@lists.airshipit.org -home-page = https://airship-deckhand.readthedocs.io/ +author_email = airship-discuss@lists.airshipit.org +home_page = https://airship-deckhand.readthedocs.io/ classifier = Intended Audience :: Information Technology @@ -13,8 +14,8 @@ classifier = Operating System :: POSIX :: Linux Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.10 [files] packages = diff --git a/setup.py b/setup.py index 0b220f45..ffec5116 100644 --- a/setup.py +++ b/setup.py @@ -12,16 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import setuptools +from setuptools import setup -# In python < 2.7.4, a lazy loading of package `pbr` will break -# setuptools if some other modules registered functions in `atexit`. -# solution from: http://bugs.python.org/issue15881#msg170215 -try: - import multiprocessing # noqa -except ImportError: - pass - -setuptools.setup( +setup( setup_requires=['pbr>=2.0.0'], - pbr=True) + pbr=True +) diff --git a/test-requirements.txt b/test-requirements.txt index 1034c706..7f46f956 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,22 +2,28 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -amqp<2.7,>=2.6.0 -coverage==4.5.1 +amqp==5.0.8 +coverage==5.5 fixtures==3.0.0 -python-subunit>=1.4.0 -os-testr==1.0.0 +python-subunit==1.4.0 +os-testr==2.0.1 testrepository==0.0.20 -testtools==2.3.0 -bandit==1.6.2 +testtools==2.5.0 +bandit==1.7.4 # NOTE(felipemonteiro): Pin here because later versions require that # content-type be present in empty responses. gabbi==1.35.1 -pifpaf==2.1.2 -oslotest==3.7.0 -yq>=2.7.2 -tox -pylibyaml~=0.1 -six>=1.15.0 -pyparsing>=2.1.0 -openstacksdk==0.36.5 +pifpaf==3.1.5 +oslotest==4.5.0 +yq==3.1.0 +tox<=4 +pylibyaml==0.1.0 +six==1.16.0 +pyparsing==2.4.7 +openstacksdk==0.59.0 +stestr==3.2.0 +requests==2.28.2 +urllib3==1.26.6 +chardet==4.0.0 +pyproject-api==1.5.0 +packaging>=23 \ No newline at end of file diff --git a/tools/gate/playbooks/osh-infra-deploy-docker.yaml b/tools/gate/playbooks/osh-infra-deploy-docker.yaml index 11638281..74d696b1 100644 --- a/tools/gate/playbooks/osh-infra-deploy-docker.yaml +++ b/tools/gate/playbooks/osh-infra-deploy-docker.yaml @@ -20,9 +20,9 @@ gather_facts: False become: yes roles: - - deploy-python + - ensure-python tags: - - deploy-python + - ensure-python - hosts: all vars_files: @@ -31,15 +31,20 @@ work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" gather_facts: True become: yes + roles: - - setup-firewall + - clear-firewall + - ensure-python + - ensure-pip + - ensure-docker + - ensure-tox - install-test-requirements - - deploy-python-pip - - deploy-docker - deploy-jq tags: - - setup-firewall + - clear-firewall + - ensure-python + - ensure-pip + - ensure-docker + - ensure-tox - install-test-requirements - - deploy-python-pip - - deploy-docker - deploy-jq diff --git a/tools/gate/playbooks/osh-infra-upgrade-host.yaml b/tools/gate/playbooks/osh-infra-upgrade-host.yaml index 3a2b79bb..b4a49c6a 100644 --- a/tools/gate/playbooks/osh-infra-upgrade-host.yaml +++ b/tools/gate/playbooks/osh-infra-upgrade-host.yaml @@ -20,9 +20,9 @@ gather_facts: False become: yes roles: - - deploy-python + - ensure-python tags: - - deploy-python + - ensure-python - hosts: all vars_files: diff --git a/tools/gate/playbooks/vars.yaml b/tools/gate/playbooks/vars.yaml index 7e45b7e1..f18a6cf6 100644 --- a/tools/gate/playbooks/vars.yaml +++ b/tools/gate/playbooks/vars.yaml @@ -12,4 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -barbican_stable_commit: 3ac3caa0138d44aa6031848d5b83802128a169b5 +# zuul_airship_deckhand_relative_path: ../airship-deckhand +# barbican_stable_commit: 3ac3caa0138d44aa6031848d5b83802128a169b5 diff --git a/tools/gate/roles/build-images/defaults/main.yaml b/tools/gate/roles/build-images/defaults/main.yaml index 7a7ad7a6..6447f48e 100644 --- a/tools/gate/roles/build-images/defaults/main.yaml +++ b/tools/gate/roles/build-images/defaults/main.yaml @@ -17,4 +17,4 @@ proxy: https: null noproxy: null -distro: ubuntu_bionic +distro: ubuntu_focal diff --git a/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml b/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml index 13e8ed00..388546d8 100644 --- a/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml +++ b/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml @@ -36,6 +36,7 @@ docker build \ --network host \ --force-rm \ + --tag deckhand \ --label zuul \ --file images/deckhand/Dockerfile.{{ distro }} \ {% if zuul_site_mirror_fqdn is defined and zuul_site_mirror_fqdn %} @@ -57,6 +58,7 @@ docker build \ --network host \ --force-rm \ + --tag deckhand \ --label zuul \ --file images/deckhand/Dockerfile.{{ distro }} \ --build-arg HTTP_PROXY="{{ proxy.http }}" \ @@ -81,4 +83,4 @@ register: airship_deckhand_image_id args: chdir: "{{ airship_deckhand_path.stdout }}" - become: yes + become: yes \ No newline at end of file diff --git a/tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml b/tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml index 765a4edd..e353c88e 100644 --- a/tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml +++ b/tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml @@ -15,7 +15,7 @@ - name: Deploy Barbican shell: | set -xe; - git reset --hard "{{ barbican_stable_commit | default('') }}" + # git reset --hard "{{ barbican_stable_commit | default('') }}" ./tools/deployment/component/barbican/barbican.sh args: chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}" diff --git a/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml b/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml index 6ffec4e4..304894d9 100644 --- a/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml +++ b/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml @@ -20,6 +20,7 @@ docker run \ --rm \ --net=host \ + --dns 10.96.0.10 \ -v "{{ deckhand_conf_dir | default('') }}":/etc/deckhand "{{ airship_deckhand_image_id.stdout }}" alembic upgrade head # Allow migrations to complete. @@ -28,10 +29,13 @@ - name: Deploy Deckhand Container shell: |- + set -ex; + docker run \ --rm \ --net=host \ -p 9000:9000 \ + --dns 10.96.0.10 \ -v "{{ deckhand_conf_dir | default('') }}":/etc/deckhand "{{ airship_deckhand_image_id.stdout }}" server & # Give the server a chance to come up. Better to poll a health check. diff --git a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml index 230bcb73..3d95ee16 100644 --- a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml +++ b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml @@ -61,6 +61,6 @@ environment: DECKHAND_TEST_URL: "127.0.0.1:9000" DECKHAND_TEST_DIR: "{{ airship_deckhand_path.stdout }}/deckhand/tests/integration/gabbits" - BARBICAN_STABLE_COMMIT: "{{ barbican_stable_commit }}" + # BARBICAN_STABLE_COMMIT: "{{ barbican_stable_commit }}" register: result failed_when: "result.rc != 0" diff --git a/tools/gate/scripts/020-deploy-postgresql.sh b/tools/gate/scripts/020-deploy-postgresql.sh index 75076ddb..a902842a 100755 --- a/tools/gate/scripts/020-deploy-postgresql.sh +++ b/tools/gate/scripts/020-deploy-postgresql.sh @@ -23,7 +23,7 @@ POSTGRES_ID=$( -e POSTGRES_DB=deckhand \ -e POSTGRES_USER=deckhand \ -e POSTGRES_PASSWORD=password \ - postgres:9.5 + postgres:14.6 ) POSTGRES_IP=$( diff --git a/tools/helm_install.sh b/tools/helm_install.sh index 6e4c042c..14b08598 100755 --- a/tools/helm_install.sh +++ b/tools/helm_install.sh @@ -17,7 +17,7 @@ set -x HELM=$1 -HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v2.17.0-linux-amd64.tar.gz"} +HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.6.3-linux-amd64.tar.gz"} function install_helm_binary { diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh index 7fffed44..34a0aa9d 100755 --- a/tools/helm_tk.sh +++ b/tools/helm_tk.sh @@ -12,58 +12,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Script to setup helm-toolkit and helm dep up the deckhand chart -# -HELM=$1 -HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"} -HTK_PATH=${HTK_PATH:-""} + + +set -eux + +HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"} HTK_STABLE_COMMIT=${HTK_COMMIT:-"f4972121bcb41c8d74748917804d2b239ab757f9"} -BUILD_DIR=${BUILD_DIR:-$(mktemp -d)} -if [[ ! -z $(echo $http_proxy) ]] -then - export no_proxy=$no_proxy,127.0.0.1 -fi +TMP_DIR=$(mktemp -d) -set -x - -function helm_serve { - if [[ -d "$HOME/.helm" ]]; then - echo ".helm directory found" - else - ${HELM} init --client-only --skip-refresh - fi - if [[ -z $(curl -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; then - ${HELM} serve & > /dev/null - while [[ -z $(curl -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; do - sleep 1 - echo "Waiting for Helm Repository" - done - else - echo "Helm serve already running" - fi - - if ${HELM} repo list | grep -q "^stable" ; then - ${HELM} repo remove stable - fi - - ${HELM} repo add local http://localhost:8879/charts +{ + HTK_REPO_DIR=$TMP_DIR/htk + git clone "$HTK_REPO" "$HTK_REPO_DIR" + (cd "$HTK_REPO_DIR" && git reset --hard "${HTK_STABLE_COMMIT}") + cp -r "${HTK_REPO_DIR}/helm-toolkit" charts/deps/ } -mkdir -p "$BUILD_DIR" -pushd "$BUILD_DIR" -git clone $HTK_REPO || true -pushd openstack-helm-infra/$HTK_PATH -git reset --hard "${HTK_STABLE_COMMIT}" - -helm_serve -# OSH Makefile is bugged, so ensure helm is in the path -if [[ ${HELM} != "helm" ]] -then - export PATH=${PATH}:$(dirname ${HELM}) -fi - -make helm-toolkit -popd && popd -${HELM} dep up charts/deckhand +rm -rf "${TMP_DIR}" diff --git a/tools/integration-tests.sh b/tools/integration-tests.sh index 8dc63161..e2b7e755 100755 --- a/tools/integration-tests.sh +++ b/tools/integration-tests.sh @@ -47,7 +47,7 @@ function deploy_osh_keystone_barbican { fi cd ${OSH_PATH} - git reset --hard ${BARBICAN_STABLE_COMMIT} + # git reset --hard ${BARBICAN_STABLE_COMMIT} # Deploy required packages ./tools/deployment/common/install-packages.sh # Deploy Kubernetes diff --git a/tools/run_pifpaf.sh b/tools/run_pifpaf.sh index 358dd9e6..b9cfef20 100755 --- a/tools/run_pifpaf.sh +++ b/tools/run_pifpaf.sh @@ -14,7 +14,7 @@ trap cleanup EXIT # variables for debugging purposes. set -ex if [ -z $(which pg_config) ]; then - sudo apt-get install libpq-dev -y + sudo apt-get install libpq-dev postgresql -y fi eval `pifpaf run postgresql` diff --git a/tox.ini b/tox.ini index dca83033..34d51ccf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,14 @@ [tox] -minversion = 2.3.1 +minversion = 3.4 +# Flag indicating to perform the packaging operation or not. +# Set it to true when using tox for an application, instead of a library. skipsdist = True -envlist = py{37,36},py{37,36}-{postgresql},functional,cover,pep8,bandit,docs +envlist = py38,py38-{postgresql},functional,cover,pep8,bandit,docs [testenv] +# Install the current package in development mode with develop mode usedevelop = True -whitelist_externals = bash +allowlist_externals = bash find rm flake8 @@ -13,7 +16,21 @@ setenv = VIRTUAL_ENV={envdir} OS_TEST_PATH=./deckhand/tests/unit LANGUAGE=en_US LC_ALL=en_US.utf-8 -passenv = OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_TEST_TIMEOUT OS_TEST_LOCK_PATH OS_TEST_PATH http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY DECKHAND_IMAGE DECKHAND_TEST_URL DECKHAND_TEST_DIR +passenv = + OS_STDOUT_CAPTURE + OS_STDERR_CAPTURE + OS_TEST_TIMEOUT + OS_TEST_LOCK_PATH + OS_TEST_PATH + http_proxy + HTTP_PROXY + https_proxy + HTTPS_PROXY + no_proxy + NO_PROXY + DECKHAND_IMAGE + DECKHAND_TEST_URL + DECKHAND_TEST_DIR deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = @@ -25,18 +42,20 @@ basepython=python3 commands = {posargs} -[testenv:py36] -basepython = python3 +[testenv:py38] commands = {[testenv]commands} stestr run {posargs} stestr slowest -[testenv:py36-postgresql] -basepython = python3 +[testenv:py38-postgresql] commands = {[testenv]commands} - {toxinidir}/tools/run_pifpaf.sh '{posargs}' + bash {toxinidir}/tools/run_pifpaf.sh '{posargs}' +allowlist_externals = + bash + find + rm [testenv:functional] basepython=python3 @@ -45,7 +64,7 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = find . -type f -name "*.pyc" -delete - stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color '{posargs}' + stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color {posargs} [testenv:functional-dev] basepython=python3 @@ -106,7 +125,7 @@ commands = # [H904] Delay string interpolations at logging calls. enable-extensions = H106,H203,H204,H205,H210,H904 # [E731] Do not assign a lambda expression, use a def. This reduces readability in some cases. -ignore = E731,F405,H405,W504 +ignore = E731,F405,H405,W504,H306 exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,releasenotes,doc,alembic/versions [testenv:docs] @@ -114,7 +133,7 @@ basepython = python3 deps = -r{toxinidir}/doc/requirements.txt commands = - {toxinidir}/tools/build-docs.sh + bash {toxinidir}/tools/build-docs.sh [testenv:releasenotes] basepython = python3 @@ -122,5 +141,5 @@ deps = -r{toxinidir}/doc/requirements.txt commands = rm -rf releasenotes/build sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html -whitelist_externals = +allowlist_externals = rm