From 093b5d2296ec5dee53631cf2eda962eb00f46107 Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Mon, 22 Jan 2018 21:44:30 +0000 Subject: [PATCH] bug(manifest): Allow specific manifest to be specified This PS allows users to specify the manifest file to use by the Armada handler by introducing a new flag called `target_manifest`. This flag was added to the API and CLI. A foundation of unit tests for the manifest handler is included in this PS. Most of the coverage is aimed at checking the various success and failure cases surrounding the new target_manifest feature. Also updates documentation to convey information about the new flag and clean up some documentation formatting inconsistencies and typos. Change-Id: I1d5a3ecc1e99b6479438d0ee5490610178be34fe --- armada/api/controller/armada.py | 86 ++++---- armada/api/controller/test.py | 9 +- armada/api/controller/validation.py | 3 +- armada/cli/apply.py | 84 +++++--- armada/cli/test.py | 14 +- armada/common/client.py | 6 +- armada/errors.py | 2 +- armada/exceptions/__init__.py | 18 ++ armada/exceptions/base_exception.py | 6 +- armada/exceptions/manifest_exceptions.py | 19 ++ armada/handlers/armada.py | 27 ++- armada/handlers/manifest.py | 123 +++++++---- .../tests/unit/api/test_armada_controller.py | 1 + armada/tests/unit/handlers/test_manifest.py | 191 ++++++++++++++++++ .../unit/resources/keystone-manifest.yaml | 135 +++++++++++++ docs/source/commands/apply.rst | 27 ++- docs/source/commands/test.rst | 12 +- docs/source/operations/guide-exceptions.rst | 10 + 18 files changed, 634 insertions(+), 139 deletions(-) create mode 100644 armada/exceptions/manifest_exceptions.py create mode 100644 armada/tests/unit/handlers/test_manifest.py create mode 100644 armada/tests/unit/resources/keystone-manifest.yaml diff --git a/armada/api/controller/armada.py b/armada/api/controller/armada.py index 639470d8..0d098898 100644 --- a/armada/api/controller/armada.py +++ b/armada/api/controller/armada.py @@ -19,57 +19,54 @@ import falcon from armada import api from armada.common import policy +from armada import exceptions from armada.handlers.armada import Armada from armada.handlers.document import ReferenceResolver from armada.handlers.override import Override class Apply(api.BaseResource): - ''' - apply armada endpoint service - ''' + """Controller for installing and updating charts defined in an Armada + manifest file. + """ + @policy.enforce('armada:create_endpoints') def on_post(self, req, resp): - try: - - # Load data from request and get options - if req.content_type == 'application/x-yaml': - data = list(self.req_yaml(req)) - if type(data[0]) is list: - documents = list(data[0]) - else: - documents = data - elif req.content_type == 'application/json': - self.logger.debug("Applying manifest based on reference.") - req_body = self.req_json(req) - doc_ref = req_body.get('hrefs', None) - - if not doc_ref: - self.logger.info("Request did not contain 'hrefs'.") - resp.status = falcon.HTTP_400 - return - - data = ReferenceResolver.resolve_reference(doc_ref) - documents = list() - for d in data: - documents.extend(list(yaml.safe_load_all(d.decode()))) - - if req_body.get('overrides', None): - overrides = Override(documents, - overrides=req_body.get('overrides')) - documents = overrides.update_manifests() + # Load data from request and get options + if req.content_type == 'application/x-yaml': + data = list(self.req_yaml(req)) + if type(data[0]) is list: + documents = list(data[0]) else: - self.error(req.context, "Unknown content-type %s" - % req.content_type) - self.return_error( - resp, - falcon.HTTP_415, - message="Request must be in application/x-yaml" - "or application/json") + documents = data + elif req.content_type == 'application/json': + self.logger.debug("Applying manifest based on reference.") + req_body = self.req_json(req) + doc_ref = req_body.get('hrefs', None) - opts = req.params + if not doc_ref: + self.logger.info("Request did not contain 'hrefs'.") + resp.status = falcon.HTTP_400 + return - # Encode filename + data = ReferenceResolver.resolve_reference(doc_ref) + documents = list() + for d in data: + documents.extend(list(yaml.safe_load_all(d.decode()))) + + if req_body.get('overrides', None): + overrides = Override(documents, + overrides=req_body.get('overrides')) + documents = overrides.update_manifests() + else: + self.error(req.context, "Unknown content-type %s" + % req.content_type) + self.return_error( + resp, + falcon.HTTP_415, + message="Request must be in application/x-yaml" + "or application/json") + try: armada = Armada( documents, disable_update_pre=req.get_param_as_bool( @@ -80,9 +77,10 @@ class Apply(api.BaseResource): 'enable_chart_cleanup'), dry_run=req.get_param_as_bool('dry_run'), wait=req.get_param_as_bool('wait'), - timeout=int(opts.get('timeout', 3600)), - tiller_host=opts.get('tiller_host', None), - tiller_port=int(opts.get('tiller_port', 44134)), + timeout=req.get_param_as_int('timeout') or 3600, + tiller_host=req.get_param('tiller_host', default=None), + tiller_port=req.get_param_as_int('tiller_port') or 44134, + target_manifest=req.get_param('target_manifest') ) msg = armada.sync() @@ -95,6 +93,8 @@ class Apply(api.BaseResource): resp.content_type = 'application/json' resp.status = falcon.HTTP_200 + except exceptions.ManifestException as e: + self.return_error(resp, falcon.HTTP_400, message=str(e)) except Exception as e: err_message = 'Failed to apply manifest: {}'.format(e) self.error(req.context, err_message) diff --git a/armada/api/controller/test.py b/armada/api/controller/test.py index 3a31fd7c..169107df 100644 --- a/armada/api/controller/test.py +++ b/armada/api/controller/test.py @@ -77,12 +77,13 @@ class Tests(api.BaseResource): @policy.enforce('armada:tests_manifest') def on_post(self, req, resp): try: - opts = req.params - tiller = Tiller(tiller_host=opts.get('tiller_host', None), - tiller_port=opts.get('tiller_port', None)) + tiller = Tiller(tiller_host=req.get_param('tiller_host', None), + tiller_port=req.get_param('tiller_port', None)) documents = self.req_yaml(req) - armada_obj = Manifest(documents).get_manifest() + target_manifest = req.get_param('target_manifest', None) + armada_obj = Manifest( + documents, target_manifest=target_manifest).get_manifest() prefix = armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) known_releases = [release[0] for release in tiller.list_charts()] diff --git a/armada/api/controller/validation.py b/armada/api/controller/validation.py index 1cb7deaf..7b6d3395 100644 --- a/armada/api/controller/validation.py +++ b/armada/api/controller/validation.py @@ -23,8 +23,7 @@ from armada.handlers.document import ReferenceResolver class Validate(api.BaseResource): - ''' - apply armada endpoint service + '''Controller for validating an Armada manifest. ''' @policy.enforce('armada:validate_manifest') diff --git a/armada/cli/apply.py b/armada/cli/apply.py index 1c3368ef..3c7e5b4b 100644 --- a/armada/cli/apply.py +++ b/armada/cli/apply.py @@ -65,46 +65,76 @@ To obtain override manifest: SHORT_DESC = "command install manifest charts" -@apply.command(name='apply', help=DESC, short_help=SHORT_DESC) +@apply.command(name='apply', + help=DESC, + short_help=SHORT_DESC) @click.argument('locations', nargs=-1) -@click.option('--api', help="Contacts service endpoint", is_flag=True) -@click.option( - '--disable-update-post', help="run charts without install", is_flag=True) -@click.option( - '--disable-update-pre', help="run charts without install", is_flag=True) -@click.option('--dry-run', help="run charts without install", is_flag=True) -@click.option( - '--enable-chart-cleanup', help="Clean up Unmanaged Charts", is_flag=True) -@click.option('--set', multiple=True, type=str, default=[]) -@click.option('--tiller-host', help="Tiller host ip") -@click.option( - '--tiller-port', help="Tiller host port", type=int, default=44134) -@click.option( - '--timeout', - help="specifies time to wait for charts", - type=int, - default=3600) -@click.option('--values', '-f', multiple=True, type=str, default=[]) -@click.option('--wait', help="wait until all charts deployed", is_flag=True) -@click.option( - '--debug/--no-debug', help='Enable or disable debugging', default=False) +@click.option('--api', + help="Contacts service endpoint.", + is_flag=True) +@click.option('--disable-update-post', + help="Disable post-update Tiller operations.", + is_flag=True) +@click.option('--disable-update-pre', + help="Disable pre-update Tiller operations.", + is_flag=True) +@click.option('--dry-run', + help="Run charts without installing them.", + is_flag=True) +@click.option('--enable-chart-cleanup', + help="Clean up unmanaged charts.", + is_flag=True) +@click.option('--set', + help=("Use to override Armada Manifest values. Accepts " + "overrides that adhere to the format ="), + multiple=True, + type=str, + default=[]) +@click.option('--tiller-host', + help="Tiller host IP.") +@click.option('--tiller-port', + help="Tiller host port.", + type=int, + default=44134) +@click.option('--timeout', + help="Specifies time to wait for charts to deploy.", + type=int, + default=3600) +@click.option('--values', + '-f', + help=("Use to override multiple Armada Manifest values by " + "reading overrides from a values.yaml-type file."), + multiple=True, + type=str, + default=[]) +@click.option('--wait', + help="Wait until all charts deployed.", + is_flag=True) +@click.option('--target-manifest', + help=('The target manifest to run. Required for specifying ' + 'which manifest to run when multiple are available.'), + default=None) +@click.option('--debug/--no-debug', + help='Enable or disable debugging.', + default=False) @click.pass_context def apply_create(ctx, locations, api, disable_update_post, disable_update_pre, dry_run, enable_chart_cleanup, set, tiller_host, tiller_port, - timeout, values, wait, debug): + timeout, values, wait, target_manifest, debug): if debug: CONF.debug = debug ApplyManifest(ctx, locations, api, disable_update_post, disable_update_pre, dry_run, enable_chart_cleanup, set, tiller_host, tiller_port, - timeout, values, wait).invoke() + timeout, values, wait, target_manifest).invoke() class ApplyManifest(CliAction): def __init__(self, ctx, locations, api, disable_update_post, disable_update_pre, dry_run, enable_chart_cleanup, set, - tiller_host, tiller_port, timeout, values, wait): + tiller_host, tiller_port, timeout, values, wait, + target_manifest): super(ApplyManifest, self).__init__() self.ctx = ctx # Filename can also be a URL reference @@ -120,6 +150,7 @@ class ApplyManifest(CliAction): self.timeout = timeout self.values = values self.wait = wait + self.target_manifest = target_manifest def output(self, resp): for result in resp: @@ -153,7 +184,8 @@ class ApplyManifest(CliAction): armada = Armada( documents, self.disable_update_pre, self.disable_update_post, self.enable_chart_cleanup, self.dry_run, self.set, self.wait, - self.timeout, self.tiller_host, self.tiller_port, self.values) + self.timeout, self.tiller_host, self.tiller_port, self.values, + self.target_manifest) resp = armada.sync() self.output(resp) diff --git a/armada/cli/test.py b/armada/cli/test.py index 29fd12c2..046a469e 100644 --- a/armada/cli/test.py +++ b/armada/cli/test.py @@ -56,14 +56,19 @@ SHORT_DESC = "command test releases" @click.option('--tiller-host', help="Tiller Host IP") @click.option( '--tiller-port', help="Tiller host Port", type=int, default=44134) +@click.option('--target-manifest', + help=('The target manifest to run. Required for specifying ' + 'which manifest to run when multiple are available.'), + default=None) @click.pass_context -def test_charts(ctx, file, release, tiller_host, tiller_port): +def test_charts(ctx, file, release, tiller_host, tiller_port, target_manifest): TestChartManifest( ctx, file, release, tiller_host, tiller_port).invoke() class TestChartManifest(CliAction): - def __init__(self, ctx, file, release, tiller_host, tiller_port): + def __init__(self, ctx, file, release, tiller_host, tiller_port, + target_manifest): super(TestChartManifest, self).__init__() self.ctx = ctx @@ -71,6 +76,7 @@ class TestChartManifest(CliAction): self.release = release self.tiller_host = tiller_host self.tiller_port = tiller_port + self.target_manifest = target_manifest def invoke(self): tiller = Tiller( @@ -107,7 +113,9 @@ class TestChartManifest(CliAction): if self.file: if not self.ctx.obj.get('api', False): documents = yaml.safe_load_all(open(self.file).read()) - armada_obj = Manifest(documents).get_manifest() + armada_obj = Manifest( + documents, + target_manifest=self.target_manifest).get_manifest() prefix = armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) diff --git a/armada/common/client.py b/armada/common/client.py index a074ffb0..56b22bc3 100644 --- a/armada/common/client.py +++ b/armada/common/client.py @@ -75,13 +75,13 @@ class ArmadaClient(object): values=None, set=None, query=None): - """Call the Armada API to apply a manifest. + """Call the Armada API to apply a Manifest. - If manifest is not None, then the request body will be a fully + If ``manifest`` is not None, then the request body will be a fully rendered set of YAML documents including overrides and values-files application. - If manifest is None and manifest_ref is not, then the request + If ``manifest`` is None and ``manifest_ref`` is not, then the request body will be a JSON structure providing a list of references to Armada manifest documents and a list of overrides. Local values files are not supported when using the API with references. diff --git a/armada/errors.py b/armada/errors.py index af7c7da3..1aeb5012 100644 --- a/armada/errors.py +++ b/armada/errors.py @@ -191,7 +191,7 @@ class AppError(Exception): """ :param description: The internal error description :param error_list: The list of errors - :param status: The desired falcon HTTP resposne code + :param status: The desired falcon HTTP response code :param title: The title of the error message :param error_list: A list of errors to be included in output messages list diff --git a/armada/exceptions/__init__.py b/armada/exceptions/__init__.py index e69de29b..203dc6b2 100644 --- a/armada/exceptions/__init__.py +++ b/armada/exceptions/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2017 The Armada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from armada.exceptions.manifest_exceptions import ManifestException + + +__all__ = ['ManifestException'] diff --git a/armada/exceptions/base_exception.py b/armada/exceptions/base_exception.py index cee250b0..f296b6e4 100644 --- a/armada/exceptions/base_exception.py +++ b/armada/exceptions/base_exception.py @@ -27,8 +27,12 @@ CONF = cfg.CONF class ArmadaBaseException(Exception): '''Base class for Armada exception and error handling.''' - def __init__(self, message=None): + def __init__(self, message=None, **kwargs): self.message = message or self.message + try: + self.message = self.message % kwargs + except TypeError: + pass super(ArmadaBaseException, self).__init__(self.message) diff --git a/armada/exceptions/manifest_exceptions.py b/armada/exceptions/manifest_exceptions.py new file mode 100644 index 00000000..a0296dc1 --- /dev/null +++ b/armada/exceptions/manifest_exceptions.py @@ -0,0 +1,19 @@ +# Copyright 2017 The Armada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from armada.exceptions import base_exception as base + + +class ManifestException(base.ArmadaBaseException): + message = 'An error occurred while generating the manifest: %(details)s.' diff --git a/armada/handlers/armada.py b/armada/handlers/armada.py index dace9955..606c3812 100644 --- a/armada/handlers/armada.py +++ b/armada/handlers/armada.py @@ -54,10 +54,23 @@ class Armada(object): timeout=DEFAULT_TIMEOUT, tiller_host=None, tiller_port=44134, - values=None): + values=None, + target_manifest=None): ''' - Initialize the Armada Engine and establish - a connection to Tiller + Initialize the Armada engine and establish a connection to Tiller. + + :param List[dict] file: Armada documents. + :param bool disable_update_pre: Disable pre-update Tiller operations. + :param bool disable_update_post: Disable post-update Tiller + operations. + :param bool enable_chart_cleanup: Clean up unmanaged charts. + :param bool dry_run: Run charts without installing them. + :param bool wait: Wait until all charts are deployed. + :param int timeout: Specifies time to wait for charts to deploy. + :param str tiller_host: Tiller host IP. + :param int tiller_port: Tiller host port. + :param str target_manifest: The target manifest to run. Useful for + specifying which manifest to run when multiple are available. ''' self.disable_update_pre = disable_update_pre self.disable_update_post = disable_update_post @@ -70,9 +83,13 @@ class Armada(object): self.values = values self.documents = file self.config = None + self.target_manifest = target_manifest def get_armada_manifest(self): - return Manifest(self.documents).get_manifest() + return Manifest( + self.documents, + target_manifest=self.target_manifest + ).get_manifest() def find_release_chart(self, known_releases, name): ''' @@ -303,7 +320,7 @@ class Armada(object): timeout=chart_timeout) if chart_wait: - # TODO(gardlt): after v0.7.1 depricate timeout values + # TODO(gardlt): after v0.7.1 deprecate timeout values if not wait_values.get('timeout', None): wait_values['timeout'] = chart_timeout diff --git a/armada/handlers/manifest.py b/armada/handlers/manifest.py index d7c27b27..63315909 100644 --- a/armada/handlers/manifest.py +++ b/armada/handlers/manifest.py @@ -12,44 +12,97 @@ # See the License for the specific language governing permissions and # limitations under the License. -from armada.const import DOCUMENT_CHART, DOCUMENT_GROUP, DOCUMENT_MANIFEST +from oslo_log import log as logging + +from armada import const +from armada import exceptions + +LOG = logging.getLogger(__name__) class Manifest(object): - def __init__(self, documents): + + def __init__(self, documents, target_manifest=None): + """Instantiates a Manifest object. + + An Armada Manifest expects that at least one of each of the following + be included in ``documents``: + + * A document with schema "armada/Chart/v1" + * A document with schema "armada/ChartGroup/v1" + + And only one document of the following is allowed: + + * A document with schema "armada/Manifest/v1" + + If multiple documents with schema "armada/Manifest/v1" are provided, + specify ``target_manifest`` to select the target one. + + :param List[dict] documents: Documents out of which to build the + Armada Manifest. + :param str target_manifest: The target manifest to use when multiple + documents with "armada/Manifest/v1" are contained in + ``documents``. Default is None. + :raises ManifestException: If the expected number of document types + are not found or if the document types are missing required + properties. + """ self.config = None self.documents = documents - self.charts = [] - self.groups = [] - self.manifest = None - self.get_documents() + self.charts, self.groups, manifests = self._find_documents( + target_manifest) - def get_documents(self): + if len(manifests) > 1: + error = ('Multiple manifests are not supported. Ensure that the ' + '`target_manifest` option is set to specify the target ' + 'manifest') + LOG.error(error) + raise exceptions.ManifestException(details=error) + else: + self.manifest = manifests[0] if manifests else None + + if not all([self.charts, self.groups, self.manifest]): + expected_schemas = [const.DOCUMENT_CHART, const.DOCUMENT_GROUP] + error = ('Documents must be a list of documents with at least one ' + 'of each of the following schemas: %s and only one ' + 'manifest' % expected_schemas) + LOG.error(error, expected_schemas) + raise exceptions.ManifestException( + details=error % expected_schemas) + + def _find_documents(self, target_manifest=None): + charts = [] + groups = [] + manifests = [] for document in self.documents: - if document.get('schema') == DOCUMENT_CHART: - self.charts.append(document) - if document.get('schema') == DOCUMENT_GROUP: - self.groups.append(document) - if document.get('schema') == DOCUMENT_MANIFEST: - self.manifest = document + if document.get('schema') == const.DOCUMENT_CHART: + charts.append(document) + if document.get('schema') == const.DOCUMENT_GROUP: + groups.append(document) + if document.get('schema') == const.DOCUMENT_MANIFEST: + manifest_name = document.get('metadata', {}).get('name') + if target_manifest: + if manifest_name == target_manifest: + manifests.append(document) + else: + manifests.append(document) + return charts, groups, manifests def find_chart_document(self, name): - try: - for chart in self.charts: - if chart.get('metadata').get('name') == name: - return chart - except Exception: - raise Exception( - "Could not find {} in {}".format(name, DOCUMENT_CHART)) + for chart in self.charts: + if chart.get('metadata', {}).get('name') == name: + return chart + raise exceptions.ManifestException( + details='Could not find a {} named "{}"'.format( + const.DOCUMENT_CHART, name)) def find_chart_group_document(self, name): - try: - for group in self.groups: - if group.get('metadata').get('name') == name: - return group - except Exception: - raise Exception( - "Could not find {} in {}".format(name, DOCUMENT_GROUP)) + for group in self.groups: + if group.get('metadata', {}).get('name') == name: + return group + raise exceptions.ManifestException( + details='Could not find a {} named "{}"'.format( + const.DOCUMENT_GROUP, name)) def build_charts_deps(self): for chart in self.charts: @@ -71,9 +124,9 @@ class Manifest(object): 'chart': chart_dep.get('data') } except Exception: - raise Exception( - "Could not find dependency chart {} in {}".format( - dep, DOCUMENT_CHART)) + raise exceptions.ManifestException( + details="Could not find dependency chart {} in {}".format( + dep, const.DOCUMENT_CHART)) def build_chart_group(self, chart_group): try: @@ -87,9 +140,9 @@ class Manifest(object): 'chart': chart_dep.get('data') } except Exception: - raise Exception( - "Could not find chart {} in {}".format( - chart, DOCUMENT_GROUP)) + raise exceptions.ManifestException( + details="Could not find chart {} in {}".format( + chart, const.DOCUMENT_GROUP)) def build_armada_manifest(self): try: @@ -106,9 +159,9 @@ class Manifest(object): self.manifest['data']['chart_groups'][iter] = ch_grp_data except Exception: - raise Exception( + raise exceptions.ManifestException( "Could not find chart group {} in {}".format( - group, DOCUMENT_MANIFEST)) + group, const.DOCUMENT_MANIFEST)) def get_manifest(self): self.build_charts_deps() diff --git a/armada/tests/unit/api/test_armada_controller.py b/armada/tests/unit/api/test_armada_controller.py index e0540c00..635a715b 100644 --- a/armada/tests/unit/api/test_armada_controller.py +++ b/armada/tests/unit/api/test_armada_controller.py @@ -52,6 +52,7 @@ class ArmadaControllerTest(base.BaseControllerTest): 'timeout': 100, 'tiller_host': None, 'tiller_port': 44134, + 'target_manifest': None } payload_url = 'http://foo.com/test.yaml' diff --git a/armada/tests/unit/handlers/test_manifest.py b/armada/tests/unit/handlers/test_manifest.py new file mode 100644 index 00000000..cb3c79ee --- /dev/null +++ b/armada/tests/unit/handlers/test_manifest.py @@ -0,0 +1,191 @@ +# Copyright 2017 The Armada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import os +import yaml + +import testtools + +from armada import const +from armada import exceptions +from armada.handlers import manifest + + +class ManifestTestCase(testtools.TestCase): + + def setUp(self): + super(ManifestTestCase, self).setUp() + examples_dir = os.path.join( + os.getcwd(), 'armada', 'tests', 'unit', 'resources') + with open(os.path.join(examples_dir, 'keystone-manifest.yaml')) as f: + self.documents = list(yaml.safe_load_all(f.read())) + + def test_get_documents(self): + armada_manifest = manifest.Manifest(self.documents) + + self.assertIsInstance(armada_manifest.charts, list) + self.assertIsInstance(armada_manifest.groups, list) + self.assertIsNotNone(armada_manifest.manifest) + + self.assertEqual(4, len(armada_manifest.charts)) + self.assertEqual(2, len(armada_manifest.groups)) + + self.assertEqual([self.documents[x] for x in range(4)], + armada_manifest.charts) + self.assertEqual([self.documents[x] for x in range(4, 6)], + armada_manifest.groups) + self.assertEqual(self.documents[-1], armada_manifest.manifest) + + def test_get_documents_with_target_manifest(self): + # Validate that specifying `target_manifest` flag returns the correct + # manifest. + armada_manifest = manifest.Manifest( + self.documents, target_manifest='armada-manifest') + + self.assertIsInstance(armada_manifest.charts, list) + self.assertIsInstance(armada_manifest.groups, list) + self.assertIsNotNone(armada_manifest.manifest) + + self.assertEqual(4, len(armada_manifest.charts)) + self.assertEqual(2, len(armada_manifest.groups)) + + self.assertEqual([self.documents[x] for x in range(4)], + armada_manifest.charts) + self.assertEqual([self.documents[x] for x in range(4, 6)], + armada_manifest.groups) + self.assertEqual(self.documents[-1], armada_manifest.manifest) + self.assertEqual('armada-manifest', + self.documents[-1]['metadata']['name']) + + def test_get_documents_with_multi_manifest_and_target_manifest(self): + # Validate that specifying `target_manifest` flag returns the correct + # manifest even if there are multiple existing manifests. (Only works + # when the manifest names are distinct or else should raise error.) + documents = copy.deepcopy(self.documents) + other_manifest = copy.deepcopy(self.documents[-1]) + other_manifest['metadata']['name'] = 'alt-armada-manifest' + documents.append(other_manifest) + + # Specify the "original" manifest and verify it works. + armada_manifest = manifest.Manifest( + documents, target_manifest='armada-manifest') + + self.assertIsInstance(armada_manifest.charts, list) + self.assertIsInstance(armada_manifest.groups, list) + self.assertIsNotNone(armada_manifest.manifest) + + self.assertEqual(4, len(armada_manifest.charts)) + self.assertEqual(2, len(armada_manifest.groups)) + + self.assertEqual([self.documents[x] for x in range(4)], + armada_manifest.charts) + self.assertEqual([self.documents[x] for x in range(4, 6)], + armada_manifest.groups) + self.assertEqual(armada_manifest.manifest, self.documents[-1]) + self.assertEqual('armada-manifest', + armada_manifest.manifest['metadata']['name']) + + # Specify the alternative manifest and verify it works. + armada_manifest = manifest.Manifest( + documents, target_manifest='alt-armada-manifest') + self.assertIsNotNone(armada_manifest.manifest) + self.assertEqual(other_manifest, armada_manifest.manifest) + self.assertEqual('alt-armada-manifest', + armada_manifest.manifest['metadata']['name']) + + def test_find_chart_document(self): + armada_manifest = manifest.Manifest(self.documents) + chart = armada_manifest.find_chart_document('helm-toolkit') + self.assertEqual(self.documents[0], chart) + + def test_find_group_document(self): + armada_manifest = manifest.Manifest(self.documents) + chart = armada_manifest.find_chart_group_document('openstack-keystone') + self.assertEqual(self.documents[-2], chart) + + +class ManifestNegativeTestCase(testtools.TestCase): + + def setUp(self): + super(ManifestNegativeTestCase, self).setUp() + examples_dir = os.path.join( + os.getcwd(), 'armada', 'tests', 'unit', 'resources') + with open(os.path.join(examples_dir, 'keystone-manifest.yaml')) as f: + self.documents = list(yaml.safe_load_all(f.read())) + + def test_get_documents_multi_manifests_raises_value_error(self): + # Validates that finding multiple manifests without `target_manifest` + # flag raises exceptions.ManifestException. + documents = copy.deepcopy(self.documents) + documents.append(documents[-1]) # Copy the last manifest. + + error_re = r'Multiple manifests are not supported.*' + self.assertRaisesRegexp( + exceptions.ManifestException, error_re, manifest.Manifest, + documents) + + def test_get_documents_multi_target_manifests_raises_value_error(self): + # Validates that finding multiple manifests with `target_manifest` + # flag raises exceptions.ManifestException. + documents = copy.deepcopy(self.documents) + documents.append(documents[-1]) # Copy the last manifest. + + error_re = r'Multiple manifests are not supported.*' + self.assertRaisesRegexp( + exceptions.ManifestException, error_re, manifest.Manifest, + documents, target_manifest='armada-manifest') + + def test_get_documents_missing_manifest(self): + # Validates exceptions.ManifestException is thrown if no manifest is + # found. Manifest is last document in sample YAML. + error_re = ('Documents must be a list of documents with at least one ' + 'of each of the following schemas: .*') + self.assertRaisesRegexp( + exceptions.ManifestException, error_re, manifest.Manifest, + self.documents[:-1]) + + def test_get_documents_missing_charts(self): + # Validates exceptions.ManifestException is thrown if no chart is + # found. Charts are first 4 documents in sample YAML. + error_re = ('Documents must be a list of documents with at least one ' + 'of each of the following schemas: .*') + self.assertRaisesRegexp( + exceptions.ManifestException, error_re, manifest.Manifest, + self.documents[4:]) + + def test_get_documents_missing_chart_groups(self): + # Validates exceptions.ManifestException is thrown if no chart is + # found. ChartGroups are 5-6 documents in sample YAML. + documents = self.documents[:4] + [self.documents[-1]] + error_re = ('Documents must be a list of documents with at least one ' + 'of each of the following schemas: .*') + self.assertRaisesRegexp( + exceptions.ManifestException, error_re, manifest.Manifest, + documents) + + def test_find_chart_document_negative(self): + armada_manifest = manifest.Manifest(self.documents) + error_re = r'Could not find a %s named "%s"' % ( + const.DOCUMENT_CHART, 'invalid') + self.assertRaisesRegexp(exceptions.ManifestException, error_re, + armada_manifest.find_chart_document, 'invalid') + + def test_find_group_document_negative(self): + armada_manifest = manifest.Manifest(self.documents) + error_re = r'Could not find a %s named "%s"' % ( + const.DOCUMENT_GROUP, 'invalid') + self.assertRaisesRegexp(exceptions.ManifestException, error_re, + armada_manifest.find_chart_group_document, + 'invalid') diff --git a/armada/tests/unit/resources/keystone-manifest.yaml b/armada/tests/unit/resources/keystone-manifest.yaml new file mode 100644 index 00000000..73fe7c19 --- /dev/null +++ b/armada/tests/unit/resources/keystone-manifest.yaml @@ -0,0 +1,135 @@ +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: helm-toolkit +data: + chart_name: helm-toolkit + release: helm-toolkit + namespace: helm-tookit + values: {} + source: + type: git + location: https://git.openstack.org/openstack/openstack-helm + subpath: helm-toolkit + reference: master + dependencies: [] +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: mariadb +data: + chart_name: mariadb + release: mariadb + namespace: openstack + timeout: 3600 + wait: + timeout: 3600 + labels: + release_group: armada-mariadb + install: + no_hooks: false + upgrade: + no_hooks: false + values: {} + source: + type: git + location: https://git.openstack.org/openstack/openstack-helm + subpath: mariadb + reference: master + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: memcached +data: + chart_name: memcached + release: memcached + namespace: openstack + timeout: 100 + wait: + timeout: 100 + labels: + release_group: armada-memcached + install: + no_hooks: false + upgrade: + no_hooks: false + values: {} + source: + type: git + location: https://git.openstack.org/openstack/openstack-helm + subpath: memcached + reference: master + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: keystone +data: + chart_name: keystone + test: true + release: keystone + namespace: openstack + timeout: 100 + wait: + timeout: 100 + labels: + release_group: armada-keystone + install: + no_hooks: false + upgrade: + no_hooks: false + pre: + delete: + - name: keystone-bootstrap + type: job + labels: + application: keystone + component: bootstrap + values: + replicas: 3 + source: + type: git + location: https://git.openstack.org/openstack/openstack-helm + subpath: keystone + reference: master + dependencies: + - helm-toolkit +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: keystone-infra-services +data: + description: "Keystone Infra Services" + sequenced: True + chart_group: + - mariadb + - memcached +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: openstack-keystone +data: + description: "Deploying OpenStack Keystone" + sequenced: True + test_charts: False + chart_group: + - keystone +--- +schema: armada/Manifest/v1 +metadata: + schema: metadata/Document/v1 + name: armada-manifest +data: + release_prefix: armada + chart_groups: + - keystone-infra-services + - openstack-keystone diff --git a/docs/source/commands/apply.rst b/docs/source/commands/apply.rst index f7865f56..f4a5611c 100644 --- a/docs/source/commands/apply.rst +++ b/docs/source/commands/apply.rst @@ -31,17 +31,22 @@ Commands $ armada apply examples/simple.yaml --values examples/simple-ovr-values.yaml Options: - --api Contacts service endpoint - --disable-update-post run charts without install - --disable-update-pre run charts without install - --dry-run run charts without install - --enable-chart-cleanup Clean up Unmanaged Charts - --set TEXT - --tiller-host TEXT Tiller host ip - --tiller-port INTEGER Tiller host port - --timeout INTEGER specifies time to wait for charts - -f, --values TEXT - --wait wait until all charts deployed + --api Contacts service endpoint. + --disable-update-post Disable post-update Tiller operations. + --disable-update-pre Disable pre-update Tiller operations. + --dry-run Run charts without installing them. + --enable-chart-cleanup Clean up unmanaged charts. + --set TEXT Use to override Armada Manifest values. Accepts + overrides that adhere to the format = + --tiller-host TEXT Tiller host IP. + --tiller-port INTEGER Tiller host port. + --timeout INTEGER Specifies time to wait for charts to deploy. + -f, --values TEXT Use to override multiple Armada Manifest values by + reading overrides from a values.yaml-type file. + --wait Wait until all charts deployed. + --target-manifest TEXT The target manifest to run. Required for specifying + which manifest to run when multiple are available. + --debug / --no-debug Enable or disable debugging. --help Show this message and exit. Synopsis diff --git a/docs/source/commands/test.rst b/docs/source/commands/test.rst index 392c8de8..aca7eede 100644 --- a/docs/source/commands/test.rst +++ b/docs/source/commands/test.rst @@ -24,11 +24,13 @@ Commands $ armada test --release blog-1 Options: - --file TEXT armada manifest - --release TEXT helm release - --tiller-host TEXT Tiller Host IP - --tiller-port INTEGER Tiller host Port - --help Show this message and exit. + --file TEXT armada manifest + --release TEXT helm release + --tiller-host TEXT Tiller Host IP + --tiller-port INTEGER Tiller host Port + --help Show this message and exit. + --target-manifest TEXT The target manifest to run. Required for specifying + which manifest to run when multiple are available. Synopsis diff --git a/docs/source/operations/guide-exceptions.rst b/docs/source/operations/guide-exceptions.rst index 37fabb3a..1874f470 100644 --- a/docs/source/operations/guide-exceptions.rst +++ b/docs/source/operations/guide-exceptions.rst @@ -89,3 +89,13 @@ Lint Exceptions +----------------------------------+------------------------------+ | InvalidArmadaObjectException | Armada object not declared. | +----------------------------------+------------------------------+ + +Manifest Exceptions +=================== ++----------------------------------+------------------------------------------------+ +| Exception | Error Description | ++==================================+================================================+ +| ManifestException | An exception occurred while attempting to build| +| | an Armada manifest. The exception will return | +| | with details as to why. | ++----------------------------------+------------------------------------------------+