diff --git a/armada/api/controller/test.py b/armada/api/controller/test.py index e66bcc7e..a3c573ae 100644 --- a/armada/api/controller/test.py +++ b/armada/api/controller/test.py @@ -133,12 +133,12 @@ class TestReleasesManifestController(api.BaseResource): armada_obj = Manifest( documents, target_manifest=target_manifest).get_manifest() - prefix = armada_obj.get(const.KEYWORD_ARMADA).get(const.KEYWORD_PREFIX) + prefix = armada_obj[const.KEYWORD_DATA][const.KEYWORD_PREFIX] known_releases = [release[0] for release in tiller.list_charts()] message = {'tests': {'passed': [], 'skipped': [], 'failed': []}} - for group in armada_obj.get(const.KEYWORD_ARMADA).get( + for group in armada_obj.get(const.KEYWORD_DATA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): chart = ch['chart'] diff --git a/armada/cli/delete.py b/armada/cli/delete.py index d142df18..bfb755d8 100644 --- a/armada/cli/delete.py +++ b/armada/cli/delete.py @@ -126,13 +126,14 @@ class DeleteChartManifest(CliAction): documents = list(yaml.safe_load_all(f.read())) try: armada_obj = Manifest(documents).get_manifest() - prefix = armada_obj.get(const.KEYWORD_ARMADA).get( + prefix = armada_obj.get(const.KEYWORD_DATA).get( const.KEYWORD_PREFIX) - for group in armada_obj.get(const.KEYWORD_ARMADA).get( + for group in armada_obj.get(const.KEYWORD_DATA).get( const.KEYWORD_GROUPS): - for ch in group.get(const.KEYWORD_CHARTS): - chart = ch.get('chart') + for ch in group.get(const.KEYWORD_DATA).get( + const.KEYWORD_CHARTS): + chart = ch.get(const.KEYWORD_DATA) release_name = release_prefixer( prefix, chart.get('release')) if release_name in known_release_names: diff --git a/armada/cli/test.py b/armada/cli/test.py index 7e83b160..55611805 100644 --- a/armada/cli/test.py +++ b/armada/cli/test.py @@ -147,10 +147,10 @@ class TestChartManifest(CliAction): armada_obj = Manifest( documents, target_manifest=self.target_manifest).get_manifest() - prefix = armada_obj.get(const.KEYWORD_ARMADA).get( + prefix = armada_obj.get(const.KEYWORD_DATA).get( const.KEYWORD_PREFIX) - for group in armada_obj.get(const.KEYWORD_ARMADA).get( + for group in armada_obj.get(const.KEYWORD_DATA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): chart = ch['chart'] diff --git a/armada/conf/__init__.py b/armada/conf/__init__.py index b6d988e3..63e87a46 100644 --- a/armada/conf/__init__.py +++ b/armada/conf/__init__.py @@ -88,7 +88,7 @@ class ChartDeployAwareLogger(logging.Logger): def _log(self, level, msg, *args, **kwargs): chart = get_current_chart() if chart: - name = chart['chart_name'] + name = chart['metadata']['name'] prefix = '[chart={}]: '.format(name) else: prefix = '' diff --git a/armada/const.py b/armada/const.py index 4a952038..0d961ee4 100644 --- a/armada/const.py +++ b/armada/const.py @@ -12,11 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Documents -DOCUMENT_CHART = 'armada/Chart/v1' -DOCUMENT_GROUP = 'armada/ChartGroup/v1' -DOCUMENT_MANIFEST = 'armada/Manifest/v1' -KEYWORD_ARMADA = 'armada' +# Keywords +KEYWORD_DATA = 'data' KEYWORD_PREFIX = 'release_prefix' KEYWORD_GROUPS = 'chart_groups' KEYWORD_CHARTS = 'chart_group' diff --git a/armada/handlers/armada.py b/armada/handlers/armada.py index d1583721..b2e567e6 100644 --- a/armada/handlers/armada.py +++ b/armada/handlers/armada.py @@ -104,13 +104,14 @@ class Armada(object): raise tiller_exceptions.TillerServicesUnavailableException() # Clone the chart sources - manifest_data = self.manifest.get(const.KEYWORD_ARMADA, {}) + manifest_data = self.manifest.get(const.KEYWORD_DATA, {}) for group in manifest_data.get(const.KEYWORD_GROUPS, []): - for ch in group.get(const.KEYWORD_CHARTS, []): + for ch in group.get(const.KEYWORD_DATA).get( + const.KEYWORD_CHARTS, []): self.get_chart(ch) def get_chart(self, ch): - chart = ch.get('chart', {}) + chart = ch.get(const.KEYWORD_DATA) chart_source = chart.get('source', {}) location = chart_source.get('location') ct_type = chart_source.get('type') @@ -158,10 +159,10 @@ class Armada(object): self.chart_cache[source_key] = repo_dir chart['source_dir'] = (self.chart_cache.get(source_key), subpath) else: - chart_name = chart.get('chart_name') - raise source_exceptions.ChartSourceException(ct_type, chart_name) + name = chart['metadata']['name'] + raise source_exceptions.ChartSourceException(ct_type, name) - for dep in ch.get('chart', {}).get('dependencies', []): + for dep in ch.get(const.KEYWORD_DATA, {}).get('dependencies', []): self.get_chart(dep) def sync(self): @@ -185,11 +186,12 @@ class Armada(object): known_releases = self.tiller.list_releases() - manifest_data = self.manifest.get(const.KEYWORD_ARMADA, {}) + manifest_data = self.manifest.get(const.KEYWORD_DATA, {}) prefix = manifest_data.get(const.KEYWORD_PREFIX) - for chartgroup in manifest_data.get(const.KEYWORD_GROUPS, []): - cg_name = chartgroup.get('name', '') + for cg in manifest_data.get(const.KEYWORD_GROUPS, []): + chartgroup = cg.get(const.KEYWORD_DATA) + cg_name = cg.get('metadata').get('name') cg_desc = chartgroup.get('description', '') cg_sequenced = chartgroup.get('sequenced', False) or self.force_wait @@ -198,11 +200,10 @@ class Armada(object): cg_desc, cg_sequenced, ' (forced)' if self.force_wait else '') - # TODO(MarshM): Deprecate the `test_charts` key + # TODO: Remove when v1 doc support is removed. cg_test_all_charts = chartgroup.get('test_charts') cg_charts = chartgroup.get(const.KEYWORD_CHARTS, []) - charts = map(lambda x: x.get('chart', {}), cg_charts) def deploy_chart(chart): set_current_chart(chart) @@ -217,7 +218,7 @@ class Armada(object): # Returns whether or not there was a failure def handle_result(chart, get_result): - name = chart['chart_name'] + name = chart['metadata']['name'] try: result = get_result() except Exception: @@ -229,7 +230,7 @@ class Armada(object): return False if cg_sequenced: - for chart in charts: + for chart in cg_charts: if (handle_result(chart, lambda: deploy_chart(chart))): break else: @@ -237,7 +238,7 @@ class Armada(object): max_workers=len(cg_charts)) as executor: future_to_chart = { executor.submit(deploy_chart, chart): chart - for chart in charts + for chart in cg_charts } for future in as_completed(future_to_chart): @@ -260,7 +261,7 @@ class Armada(object): if self.enable_chart_cleanup: self._chart_cleanup( prefix, - self.manifest[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS], msg) + self.manifest[const.KEYWORD_DATA][const.KEYWORD_GROUPS], msg) LOG.info('Done applying manifest.') return msg diff --git a/armada/handlers/chart_deploy.py b/armada/handlers/chart_deploy.py index 139070d7..64f9045d 100644 --- a/armada/handlers/chart_deploy.py +++ b/armada/handlers/chart_deploy.py @@ -41,7 +41,8 @@ class ChartDeploy(object): self.timeout = timeout self.tiller = tiller - def execute(self, chart, cg_test_all_charts, prefix, known_releases): + def execute(self, ch, cg_test_all_charts, prefix, known_releases): + chart = ch[const.KEYWORD_DATA] namespace = chart.get('namespace') release = chart.get('release') release_name = r.release_prefixer(prefix, release) @@ -73,7 +74,7 @@ class ChartDeploy(object): # Begin Chart timeout deadline deadline = time.time() + chart_wait.get_timeout() - chartbuilder = ChartBuilder(chart) + chartbuilder = ChartBuilder(ch) new_chart = chartbuilder.get_helm_chart() # TODO(mark-burnett): It may be more robust to directly call diff --git a/armada/handlers/chartbuilder.py b/armada/handlers/chartbuilder.py index d74b2c0b..903a57aa 100644 --- a/armada/handlers/chartbuilder.py +++ b/armada/handlers/chartbuilder.py @@ -25,6 +25,7 @@ from oslo_config import cfg from oslo_log import log as logging from armada.exceptions import chartbuilder_exceptions +from armada import const LOG = logging.getLogger(__name__) @@ -49,6 +50,7 @@ class ChartBuilder(object): # store chart schema self.chart = chart + self.chart_data = chart[const.KEYWORD_DATA] # extract, pull, whatever the chart from its source self.source_directory = self.get_source_path() @@ -62,7 +64,7 @@ class ChartBuilder(object): Returns "/" taken from the "source_dir" property from the chart, or else "" if the property isn't a 2-tuple. ''' - source_dir = self.chart.get('source_dir') + source_dir = self.chart_data.get('source_dir') return (os.path.join(*source_dir) if (source_dir and isinstance(source_dir, (list, tuple)) and len(source_dir) == 2) else "") @@ -206,7 +208,7 @@ class ChartBuilder(object): Process all files in templates/ as a template to attach to the chart, building a :class:`hapi.chart.template_pb2.Template` object. ''' - chart_name = self.chart.get('chart_name') + chart_name = self.chart['metadata']['name'] templates = [] if not os.path.exists( os.path.join(self.source_directory, 'templates')): @@ -240,12 +242,11 @@ class ChartBuilder(object): return self._helm_chart dependencies = [] - chart_dependencies = self.chart.get('dependencies', []) - chart_name = self.chart.get('chart_name', None) - chart_release = self.chart.get('release', None) - for dep in chart_dependencies: - dep_chart = dep.get('chart', {}) - dep_chart_name = dep_chart.get('chart_name', None) + chart_dependencies = self.chart_data.get('dependencies', []) + chart_name = self.chart['metadata']['name'] + chart_release = self.chart_data.get('release', None) + for dep_chart in chart_dependencies: + dep_chart_name = dep_chart['metadata']['name'] LOG.info("Building dependency chart %s for release %s.", dep_chart_name, chart_release) try: diff --git a/armada/handlers/manifest.py b/armada/handlers/manifest.py index a030ff67..df6d4f7e 100644 --- a/armada/handlers/manifest.py +++ b/armada/handlers/manifest.py @@ -17,6 +17,7 @@ from oslo_log import log as logging from armada import const from armada import exceptions +from armada.handlers import schema LOG = logging.getLogger(__name__) @@ -62,10 +63,10 @@ class Manifest(object): self.manifest = manifests[0] if manifests else None if not all([self.charts, self.groups, self.manifest]): - expected_schemas = [const.DOCUMENT_CHART, const.DOCUMENT_GROUP] - error = ('Documents must be a list of documents with at least one ' - 'of each of the following schemas: %s and only one ' - 'manifest' % expected_schemas) + expected_schemas = [schema.TYPE_CHART, schema.TYPE_CHARTGROUP] + error = ('Documents must include at least one of each of {} ' + 'and only one {}').format(expected_schemas, + schema.TYPE_MANIFEST) LOG.error(error) raise exceptions.ManifestException(details=error) @@ -87,11 +88,14 @@ class Manifest(object): groups = [] manifests = [] for document in self.documents: - if document.get('schema') == const.DOCUMENT_CHART: + schema_info = schema.get_schema_info(document.get('schema')) + if not schema_info: + continue + if schema_info.type == schema.TYPE_CHART: charts.append(document) - if document.get('schema') == const.DOCUMENT_GROUP: + if schema_info.type == schema.TYPE_CHARTGROUP: groups.append(document) - if document.get('schema') == const.DOCUMENT_MANIFEST: + if schema_info.type == schema.TYPE_MANIFEST: manifest_name = document.get('metadata', {}).get('name') if target_manifest: if manifest_name == target_manifest: @@ -113,8 +117,8 @@ class Manifest(object): if chart.get('metadata', {}).get('name') == name: return chart raise exceptions.BuildChartException( - details='Could not build {} named "{}"'.format( - const.DOCUMENT_CHART, name)) + details='Could not find {} named "{}"'.format( + schema.TYPE_CHART, name)) def find_chart_group_document(self, name): """Returns a chart group document with the specified name @@ -129,8 +133,8 @@ class Manifest(object): if group.get('metadata', {}).get('name') == name: return group raise exceptions.BuildChartGroupException( - details='Could not build {} named "{}"'.format( - const.DOCUMENT_GROUP, name)) + details='Could not find {} named "{}"'.format( + schema.TYPE_CHARTGROUP, name)) def build_chart_deps(self, chart): """Recursively build chart dependencies for ``chart``. @@ -143,20 +147,19 @@ class Manifest(object): under ``chart['data']['dependencies']`` could not be found. """ try: - chart_dependencies = chart.get('data', {}).get('dependencies', []) + chart_dependencies = chart.get(const.KEYWORD_DATA, {}).get( + 'dependencies', []) for iter, dep in enumerate(chart_dependencies): if isinstance(dep, dict): continue chart_dep = self.find_chart_document(dep) self.build_chart_deps(chart_dep) - chart['data']['dependencies'][iter] = { - 'chart': chart_dep.get('data', {}) - } + chart[const.KEYWORD_DATA]['dependencies'][iter] = chart_dep except Exception: raise exceptions.ChartDependencyException( - details="Could not build dependencies for chart {} in {}". - format( - chart.get('metadata').get('name'), const.DOCUMENT_CHART)) + details='Could not build dependencies for {} named "{}"'. + format(schema.TYPE_CHART, + chart.get('metadata').get('name'))) else: return chart @@ -173,19 +176,19 @@ class Manifest(object): try: chart = None for iter, chart in enumerate( - chart_group.get('data', {}).get('chart_group', [])): + chart_group.get(const.KEYWORD_DATA).get( + const.KEYWORD_CHARTS, [])): if isinstance(chart, dict): continue - chart_dep = self.find_chart_document(chart) - self.build_chart_deps(chart_dep) - chart_group['data']['chart_group'][iter] = { - 'chart': chart_dep.get('data', {}) - } + chart_object = self.find_chart_document(chart) + self.build_chart_deps(chart_object) + chart_group[const.KEYWORD_DATA][const.KEYWORD_CHARTS][iter] = \ + chart_object except exceptions.ManifestException: cg_name = chart_group.get('metadata', {}).get('name') raise exceptions.BuildChartGroupException( - details="Could not build chart group {} in {}".format( - cg_name, const.DOCUMENT_GROUP)) + details='Could not build {} named "{}"'.format( + schema.TYPE_CHARTGROUP, cg_name)) return chart_group @@ -196,20 +199,18 @@ class Manifest(object): :returns: The Armada manifest with the data of the chart groups. :rtype: dict :raises ManifestException: If a chart group's data listed - under ``chart_group['data']`` could not be found. + under ``chart_group[const.KEYWORD_DATA]`` could not be found. """ for iter, group in enumerate( - self.manifest.get('data', {}).get('chart_groups', [])): + self.manifest.get(const.KEYWORD_DATA, {}).get( + const.KEYWORD_GROUPS, [])): if isinstance(group, dict): continue chart_grp = self.find_chart_group_document(group) self.build_chart_group(chart_grp) - # Add name to chart group - ch_grp_data = chart_grp.get('data', {}) - ch_grp_data['name'] = chart_grp.get('metadata', {}).get('name') - - self.manifest['data']['chart_groups'][iter] = ch_grp_data + self.manifest[const.KEYWORD_DATA][const.KEYWORD_GROUPS][iter] = \ + chart_grp return self.manifest @@ -221,4 +222,4 @@ class Manifest(object): """ self.build_armada_manifest() - return {'armada': self.manifest.get('data', {})} + return self.manifest diff --git a/armada/handlers/override.py b/armada/handlers/override.py index 3250711e..cd7a4a4f 100644 --- a/armada/handlers/override.py +++ b/armada/handlers/override.py @@ -16,9 +16,9 @@ import collections import json import yaml -from armada import const from armada.exceptions import override_exceptions from armada.exceptions import validate_exceptions +from armada.handlers import schema from armada.utils import validate @@ -65,17 +65,18 @@ class Override(object): def find_document_type(self, alias): if alias == 'chart_group': - return const.DOCUMENT_GROUP + return schema.TYPE_CHARTGROUP if alias == 'chart': - return const.DOCUMENT_CHART + return schema.TYPE_CHART if alias == 'manifest': - return const.DOCUMENT_MANIFEST + return schema.TYPE_MANIFEST else: raise ValueError("Could not find {} document".format(alias)) def find_manifest_document(self, doc_path): for doc in self.documents: - if doc.get('schema') == self.find_document_type( + schema_info = schema.get_schema_info(doc.get('schema')) + if schema_info.type == self.find_document_type( doc_path[0]) and doc.get('metadata', {}).get('name') == doc_path[1]: return doc @@ -121,45 +122,29 @@ class Override(object): new_data = self.array_to_dict(data_path, new_value) self.update(document.get('data', {}), new_data) - def update_document(self, merging_values): + def update_documents(self, merging_values): for doc in merging_values: - if doc.get('schema') == const.DOCUMENT_CHART: - self.update_chart_document(doc) - if doc.get('schema') == const.DOCUMENT_GROUP: - self.update_chart_group_document(doc) - if doc.get('schema') == const.DOCUMENT_MANIFEST: - self.update_armada_manifest(doc) + self.update_document(doc) - def update_chart_document(self, ovr): - for doc in self.documents: - if doc.get('schema') == const.DOCUMENT_CHART and doc.get( - 'metadata', {}).get('name') == ovr.get('metadata', - {}).get('name'): - self.update(doc.get('data', {}), ovr.get('data', {})) - return - - def update_chart_group_document(self, ovr): - for doc in self.documents: - if doc.get('schema') == const.DOCUMENT_GROUP and doc.get( - 'metadata', {}).get('name') == ovr.get('metadata', - {}).get('name'): - self.update(doc.get('data', {}), ovr.get('data', {})) - return - - def update_armada_manifest(self, ovr): - for doc in self.documents: - if doc.get('schema') == const.DOCUMENT_MANIFEST and doc.get( - 'metadata', {}).get('name') == ovr.get('metadata', - {}).get('name'): - self.update(doc.get('data', {}), ovr.get('data', {})) - return + def update_document(self, ovr): + ovr_schema_info = schema.get_schema_info(ovr.get('schema')) + if ovr_schema_info: + for doc in self.documents: + schema_info = schema.get_schema_info(doc.get('schema')) + if schema_info: + if schema_info == ovr_schema_info: + if doc['metadata']['name'] == ovr['metadata']['name']: + data = doc.get('data', {}) + ovr_data = ovr.get('data', {}) + self.update(data, ovr_data) + return def update_manifests(self): if self.values: for value in self.values: merging_values = self._load_yaml_file(value) - self.update_document(merging_values) + self.update_documents(merging_values) # Validate document with updated values self._document_checker(self.documents, self.values) diff --git a/armada/handlers/schema.py b/armada/handlers/schema.py new file mode 100644 index 00000000..1fd50e92 --- /dev/null +++ b/armada/handlers/schema.py @@ -0,0 +1,80 @@ +# Copyright 2019 The Armada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pkg_resources +import re +import yaml + +# Types +TYPE_CHART = 'Chart' +TYPE_CHARTGROUP = 'ChartGroup' +TYPE_MANIFEST = 'Manifest' + +# Versions +VERSION_FORMAT = r'^v(\d+)$' +VERSION_MIN = 1 +VERSION_MAX = 2 + +# Creates a mapping between ``metadata.name``: ``data`` where the +# ``metadata.name`` is the ``schema`` of a manifest and the ``data`` is the +# JSON schema to be used to validate the manifest in question. +_SCHEMAS = {} + + +class SchemaInfo(object): + + def __init__(self, type, version, data): + self.type = type + self.version = version + self.data = data + + def __eq__(self, other): + return self.type == other.type and self.version == other.version + + +def get_schema_info(name): + return _SCHEMAS.get(name) + + +def _get_schema_info(name, data): + parts = name.split('/') + prefix, type, version_string = parts + version_match = re.search(VERSION_FORMAT, version_string) + version = int(version_match.group(1)) + return SchemaInfo(type, version, data) + + +def _get_schema_dir(): + return pkg_resources.resource_filename('armada', 'schemas') + + +def _load_schemas(): + """Populates ``_SCHEMAS`` with the schemas defined in package + ``armada.schemas``. + + """ + schema_dir = _get_schema_dir() + for schema_file in os.listdir(schema_dir): + with open(os.path.join(schema_dir, schema_file)) as f: + for schema in yaml.safe_load_all(f): + name = schema['metadata']['name'] + if name in _SCHEMAS: + raise RuntimeError( + 'Duplicate schema specified for: %s.' % name) + _SCHEMAS[name] = _get_schema_info(name, schema['data']) + + +# Fill the cache. +_load_schemas() diff --git a/armada/handlers/test.py b/armada/handlers/test.py index 5220f036..23f71ae7 100644 --- a/armada/handlers/test.py +++ b/armada/handlers/test.py @@ -60,9 +60,7 @@ class Test(object): self.timeout = const.DEFAULT_TEST_TIMEOUT - # NOTE(drewwalters96): Support the chart_group `test_charts` key until - # its deprecation period ends. The `test.enabled`, `enable_all` flag, - # and deprecated, boolean `test` key override this value if provided. + # TODO: Remove when v1 doc support is removed. if cg_test_charts is not None: LOG.warn('Chart group key `test_charts` is deprecated and will be ' 'removed. Use `test.enabled` instead.') @@ -70,7 +68,7 @@ class Test(object): else: self.test_enabled = True - # NOTE: Support old, boolean `test` key until deprecation period ends. + # TODO: Remove when v1 doc support is removed. if (type(test_values) == bool): LOG.warn('Boolean value for chart `test` key is deprecated and ' 'will be removed. Use `test.enabled` instead.') diff --git a/armada/handlers/tiller.py b/armada/handlers/tiller.py index 4c99738f..9499a6f5 100644 --- a/armada/handlers/tiller.py +++ b/armada/handlers/tiller.py @@ -30,8 +30,10 @@ from oslo_config import cfg from oslo_log import log as logging from armada import const +from armada.conf import get_current_chart from armada.exceptions import tiller_exceptions as ex from armada.handlers.k8s import K8s +from armada.handlers import schema from armada.utils import helm from armada.utils.release import label_selectors, get_release_status @@ -303,6 +305,7 @@ class Tiller(object): :param namespace: name of pod for actions ''' + # TODO: Remove when v1 doc support is removed. try: for action in actions.get('update', []): name = action.get('name') @@ -667,15 +670,20 @@ class Tiller(object): self.k8s.delete_job_action(jb_name, namespace, timeout=timeout) handled = True - if resource_type == 'cronjob' or resource_type == 'job': + # TODO: Remove when v1 doc support is removed. + chart = get_current_chart() + schema_info = schema.get_schema_info(chart['schema']) + job_implies_cronjob = schema_info.version < 2 + implied_cronjob = resource_type == 'job' and job_implies_cronjob + + if resource_type == 'cronjob' or implied_cronjob: get_jobs = self.k8s.get_namespace_cron_job( namespace, label_selector=label_selector) for jb in get_jobs.items: jb_name = jb.metadata.name - if resource_type == 'job': - # TODO: Eventually disallow this, allowing initially since - # some existing clients were expecting this behavior. + # TODO: Remove when v1 doc support is removed. + if implied_cronjob: LOG.warn("Deleting cronjobs via `type: job` is " "deprecated, use `type: cronjob` instead") @@ -726,7 +734,7 @@ class Tiller(object): values, timeout=const.DEFAULT_TILLER_TIMEOUT): ''' - update statefullsets (daemon, stateful) + update statefulsets (daemon, stateful) ''' if action_type == 'daemonset': diff --git a/armada/schemas/armada-chart-schema.yaml b/armada/schemas/armada-chart-schema-v1.yaml similarity index 93% rename from armada/schemas/armada-chart-schema.yaml rename to armada/schemas/armada-chart-schema-v1.yaml index 353eedc0..a8814fa7 100644 --- a/armada/schemas/armada-chart-schema.yaml +++ b/armada/schemas/armada-chart-schema-v1.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# JSON schema for validating Armada charts. +# NOTE: Do not modify this schema, it is deprecated. --- schema: deckhand/DataSchema/v1 metadata: @@ -60,7 +60,6 @@ data: additionalProperties: false test: anyOf: - # TODO: Remove boolean support after deprecation period. - type: boolean - type: object properties: @@ -75,7 +74,6 @@ data: type: boolean additionalProperties: false additionalProperties: false - # TODO(MarshM): Deprecate this `timeout` in favor of `wait.timeout` timeout: type: integer wait: @@ -153,8 +151,6 @@ data: $ref: '#/definitions/hook_action' create: $ref: '#/definitions/hook_action' - # TODO(drewwalters96): Armada ignores post-update actions. Remove them - # in future schemas. post: type: object additionalProperties: false diff --git a/armada/schemas/armada-chart-schema-v2.yaml b/armada/schemas/armada-chart-schema-v2.yaml new file mode 100644 index 00000000..49e020cd --- /dev/null +++ b/armada/schemas/armada-chart-schema-v2.yaml @@ -0,0 +1,151 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# JSON schema for validating Armada charts. +--- +schema: deckhand/DataSchema/v1 +metadata: + name: armada/Chart/v2 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + definitions: + labels: + type: object + additionalProperties: + type: string + hook_action: + type: array + items: + properties: + type: + type: string + labels: + $ref: '#/definitions/labels' + required: + - type + additionalProperties: false + type: object + properties: + release: + type: string + namespace: + type: string + values: + type: object + # TODO: Remove this, and just read dependencies out of `chart` dir as helm + # CLI does. + dependencies: + type: array + items: + type: string + protected: + type: object + properties: + continue_processing: + type: boolean + additionalProperties: false + test: + type: object + properties: + enabled: + type: boolean + timeout: + type: integer + options: + type: object + properties: + cleanup: + type: boolean + additionalProperties: false + additionalProperties: false + wait: + type: object + properties: + timeout: + type: integer + resources: + type: array + items: + properties: + type: + type: string + labels: + $ref: '#/definitions/labels' + min_ready: + anyOf: + - type: integer + - type: string + required: + - type + additionalProperties: false + labels: + $ref: "#/definitions/labels" + # Config for helm's native `--wait` param. + native: + type: object + properties: + enabled: + type: boolean + additionalProperties: false + additionalProperties: false + source: + type: object + properties: + type: + type: string + location: + type: string + subpath: + type: string + reference: + type: string + proxy_server: + type: string + auth_method: + type: string + required: + - location + - type + delete: + type: object + properties: + timeout: + type: integer + upgrade: + type: object + properties: + no_hooks: + type: boolean + pre: + type: object + additionalProperties: false + properties: + delete: + $ref: '#/definitions/hook_action' + options: + type: object + properties: + force: + type: boolean + recreate_pods: + type: boolean + additionalProperties: false + additionalProperties: false + required: + - namespace + - release + - source + additionalProperties: false +... diff --git a/armada/schemas/armada-chartgroup-schema.yaml b/armada/schemas/armada-chartgroup-schema-v1.yaml similarity index 95% rename from armada/schemas/armada-chartgroup-schema.yaml rename to armada/schemas/armada-chartgroup-schema-v1.yaml index a7e66ddd..4f0c59bb 100644 --- a/armada/schemas/armada-chartgroup-schema.yaml +++ b/armada/schemas/armada-chartgroup-schema-v1.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# JSON schema for validating Armada chart groups. +# NOTE: Do not modify this schema, it is deprecated. --- schema: deckhand/DataSchema/v1 metadata: diff --git a/armada/schemas/armada-chartgroup-schema-v2.yaml b/armada/schemas/armada-chartgroup-schema-v2.yaml new file mode 100644 index 00000000..6492a649 --- /dev/null +++ b/armada/schemas/armada-chartgroup-schema-v2.yaml @@ -0,0 +1,38 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# JSON schema for validating Armada chart groups. +--- +schema: deckhand/DataSchema/v1 +metadata: + name: armada/ChartGroup/v2 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + properties: + name: + type: string + description: + type: string + sequenced: + type: boolean + chart_group: + type: array + items: + type: string + required: + # TODO: Rename to `charts`? + - chart_group + additionalProperties: false +... diff --git a/armada/schemas/armada-manifest-schema-v1.yaml b/armada/schemas/armada-manifest-schema-v1.yaml new file mode 100644 index 00000000..66a10757 --- /dev/null +++ b/armada/schemas/armada-manifest-schema-v1.yaml @@ -0,0 +1,34 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: Do not modify this schema, it is deprecated. +--- +schema: deckhand/DataSchema/v1 +metadata: + name: armada/Manifest/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + properties: + release_prefix: + type: string + chart_groups: + type: array + items: + type: string + required: + - chart_groups + - release_prefix + additionalProperties: false +... diff --git a/armada/schemas/armada-manifest-schema.yaml b/armada/schemas/armada-manifest-schema-v2.yaml similarity index 97% rename from armada/schemas/armada-manifest-schema.yaml rename to armada/schemas/armada-manifest-schema-v2.yaml index 4bdd677c..6e7f11cd 100644 --- a/armada/schemas/armada-manifest-schema.yaml +++ b/armada/schemas/armada-manifest-schema-v2.yaml @@ -16,7 +16,7 @@ --- schema: deckhand/DataSchema/v1 metadata: - name: armada/Manifest/v1 + name: armada/Manifest/v2 schema: metadata/Control/v1 data: $schema: http://json-schema.org/schema# diff --git a/armada/tests/unit/api/test_test_controller.py b/armada/tests/unit/api/test_test_controller.py index 140e19e4..335de7cb 100644 --- a/armada/tests/unit/api/test_test_controller.py +++ b/armada/tests/unit/api/test_test_controller.py @@ -169,8 +169,7 @@ class TestReleasesManifestControllerNegativeTest(base.BaseControllerTest): self.assertIn({ 'message': ('An error occurred while building chart group: ' - 'Could not build chart group keystone-infra-services in ' - 'armada/ChartGroup/v1.'), + 'Could not build ChartGroup named "keystone-infra-services".'), 'error': True, 'kind': diff --git a/armada/tests/unit/handlers/test_armada.py b/armada/tests/unit/handlers/test_armada.py index 2be30682..1a280690 100644 --- a/armada/tests/unit/handlers/test_armada.py +++ b/armada/tests/unit/handlers/test_armada.py @@ -158,125 +158,153 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase): armada_obj.pre_flight_ops() expected_config = { - 'armada': { - 'release_prefix': - 'armada', + 'schema': 'armada/Manifest/v1', + 'metadata': { + 'schema': 'metadata/Document/v1', + 'name': 'example-manifest' + }, + 'data': { + 'release_prefix': 'armada', 'chart_groups': [{ - 'chart_group': [{ - 'chart': { - 'dependencies': [], - 'chart_name': 'test_chart_1', - 'namespace': 'test', - 'release': 'test_chart_1', - 'source': { - 'location': ('git://github.com/dummy/armada'), - 'reference': 'master', - 'subpath': 'chart_1', - 'type': 'git' + 'schema': 'armada/ChartGroup/v1', + 'metadata': { + 'schema': 'metadata/Document/v1', + 'name': 'example-group' + }, + 'data': { + 'chart_group': [{ + 'schema': 'armada/Chart/v1', + 'metadata': { + 'schema': 'metadata/Document/v1', + 'name': 'example-chart-1' }, - 'source_dir': CHART_SOURCES[0], - 'values': {}, - 'wait': { - 'timeout': 10, - 'native': { - 'enabled': False - } - }, - 'test': { - 'enabled': True - } - } - }, { - 'chart': { - 'dependencies': [], - 'chart_name': 'test_chart_2', - 'namespace': 'test', - 'protected': { - 'continue_processing': True - }, - 'release': 'test_chart_2', - 'source': { - 'location': '/tmp/dummy/armada', - 'subpath': 'chart_2', - 'type': 'local' - }, - 'source_dir': CHART_SOURCES[1], - 'values': {}, - 'wait': { - 'timeout': 10 - }, - 'upgrade': { - 'no_hooks': False, - 'options': { - 'force': True, - 'recreate_pods': True - } - }, - 'test': { - 'enabled': True, - 'options': { - 'cleanup': True + 'data': { + 'dependencies': [], + 'chart_name': 'test_chart_1', + 'namespace': 'test', + 'release': 'test_chart_1', + 'source': { + 'location': + 'git://github.com/dummy/armada', + 'reference': 'master', + 'subpath': 'chart_1', + 'type': 'git' + }, + 'source_dir': CHART_SOURCES[0], + 'values': {}, + 'wait': { + 'timeout': 10, + 'native': { + 'enabled': False + } + }, + 'test': { + 'enabled': True } } - } - }, { - 'chart': { - 'dependencies': [], - 'chart_name': 'test_chart_3', - 'namespace': 'test', - 'protected': { - 'continue_processing': False + }, { + 'schema': 'armada/Chart/v1', + 'metadata': { + 'schema': 'metadata/Document/v1', + 'name': 'example-chart-2' }, - 'release': 'test_chart_3', - 'source': { - 'location': '/tmp/dummy/armada', - 'subpath': 'chart_3', - 'type': 'local' - }, - 'source_dir': CHART_SOURCES[2], - 'values': {}, - 'wait': { - 'timeout': 10 - }, - 'upgrade': { - 'no_hooks': False + 'data': { + 'dependencies': [], + 'chart_name': 'test_chart_2', + 'namespace': 'test', + 'protected': { + 'continue_processing': True + }, + 'release': 'test_chart_2', + 'source': { + 'location': '/tmp/dummy/armada', + 'subpath': 'chart_2', + 'type': 'local' + }, + 'source_dir': CHART_SOURCES[1], + 'values': {}, + 'wait': { + 'timeout': 10 + }, + 'upgrade': { + 'no_hooks': False, + 'options': { + 'force': True, + 'recreate_pods': True + } + }, + 'test': { + 'enabled': True, + 'options': { + 'cleanup': True + } + } } - } - }, { - 'chart': { - 'dependencies': [], - 'chart_name': 'test_chart_4', - 'namespace': 'test', - 'release': 'test_chart_4', - 'source': { - 'location': '/tmp/dummy/armada', - 'subpath': 'chart_4', - 'type': 'local' + }, { + 'schema': 'armada/Chart/v1', + 'metadata': { + 'schema': 'metadata/Document/v1', + 'name': 'example-chart-3' }, - 'source_dir': CHART_SOURCES[3], - 'values': {}, - 'wait': { - 'timeout': 10 + 'data': { + 'dependencies': [], + 'chart_name': 'test_chart_3', + 'namespace': 'test', + 'protected': { + 'continue_processing': False + }, + 'release': 'test_chart_3', + 'source': { + 'location': '/tmp/dummy/armada', + 'subpath': 'chart_3', + 'type': 'local' + }, + 'source_dir': CHART_SOURCES[2], + 'values': {}, + 'wait': { + 'timeout': 10 + }, + 'upgrade': { + 'no_hooks': False + } + } + }, { + 'schema': 'armada/Chart/v1', + 'metadata': { + 'schema': 'metadata/Document/v1', + 'name': 'example-chart-4' }, - 'upgrade': { - 'no_hooks': False - }, - 'test': True - } - }], - 'description': - 'this is a test', - 'name': - 'example-group', - 'sequenced': - True + 'data': { + 'dependencies': [], + 'chart_name': 'test_chart_4', + 'namespace': 'test', + 'release': 'test_chart_4', + 'source': { + 'location': '/tmp/dummy/armada', + 'subpath': 'chart_4', + 'type': 'local' + }, + 'source_dir': CHART_SOURCES[3], + 'values': {}, + 'wait': { + 'timeout': 10 + }, + 'upgrade': { + 'no_hooks': False + }, + 'test': True + } + }], + 'description': 'this is a test', + 'sequenced': True + } }] } } # yapf: disable self.assertTrue(hasattr(armada_obj, 'manifest')) self.assertIsInstance(armada_obj.manifest, dict) - self.assertIn('armada', armada_obj.manifest) + self.assertIn('data', armada_obj.manifest) self.assertEqual(expected_config, armada_obj.manifest) @mock.patch.object(armada, 'source') @@ -314,9 +342,11 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase): armada_obj.post_flight_ops() - for group in armada_obj.manifest['armada']['chart_groups']: - for counter, chart in enumerate(group.get('chart_group')): - if chart.get('chart').get('source').get('type') == 'git': + for group in armada_obj.manifest['data']['chart_groups']: + for counter, chart in enumerate( + group.get(const.KEYWORD_DATA).get(const.KEYWORD_CHARTS)): + if chart.get( + const.KEYWORD_DATA).get('source').get('type') == 'git': mock_source.source_cleanup.assert_called_with( CHART_SOURCES[counter][0]) @@ -348,7 +378,8 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase): armada_obj = armada.Armada(yaml_documents, m_tiller) armada_obj.chart_deploy.get_diff = mock.Mock() - chart_group = armada_obj.manifest['armada']['chart_groups'][0] + cg = armada_obj.manifest['data']['chart_groups'][0] + chart_group = cg['data'] charts = chart_group['chart_group'] cg_test_all_charts = chart_group.get('test_charts') @@ -380,9 +411,9 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase): expected_test_constructor_calls = [] for c in charts: - chart = c['chart'] + chart = c['data'] release = chart['release'] - prefix = armada_obj.manifest['armada']['release_prefix'] + prefix = armada_obj.manifest['data']['release_prefix'] release_name = release_prefixer(prefix, release) # Simplified check because the actual code uses logical-or's # multiple conditions, so this is enough. @@ -394,8 +425,8 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase): mock.call( mock_chartbuilder().get_helm_chart(), "{}-{}".format( - armada_obj.manifest['armada'] - ['release_prefix'], chart['release']), + armada_obj.manifest['data']['release_prefix'], + chart['release']), chart['namespace'], values=yaml.safe_dump(chart['values']), wait=native_wait_enabled, @@ -420,7 +451,7 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase): mock.call( mock_chartbuilder().get_helm_chart(), "{}-{}".format( - armada_obj.manifest['armada'] + armada_obj.manifest['data'] ['release_prefix'], chart['release']), chart['namespace'], @@ -449,7 +480,7 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase): mock.call( mock_chartbuilder().get_helm_chart(), "{}-{}".format( - armada_obj.manifest['armada'] + armada_obj.manifest['data'] ['release_prefix'], chart['release']), chart['namespace'], @@ -647,8 +678,8 @@ class ArmadaNegativeHandlerTestCase(base.ArmadaTestCase): def test_armada_get_manifest_exception(self, mock_source): """Test armada handling with invalid manifest.""" yaml_documents = list(yaml.safe_load_all(TEST_YAML)) - error_re = ('Documents must be a list of documents with at least one ' - 'of each of the following schemas: .*') + error_re = ('.*Documents must include at least one of each of .* and ' + 'only one .*') self.assertRaisesRegexp(ManifestException, error_re, armada.Armada, yaml_documents[:1], mock.MagicMock()) diff --git a/armada/tests/unit/handlers/test_chartbuilder.py b/armada/tests/unit/handlers/test_chartbuilder.py index c499a67d..b2e3fb88 100644 --- a/armada/tests/unit/handlers/test_chartbuilder.py +++ b/armada/tests/unit/handlers/test_chartbuilder.py @@ -23,6 +23,7 @@ from hapi.chart.metadata_pb2 import Metadata import mock import testtools +from armada import const from armada.handlers.chartbuilder import ChartBuilder from armada.exceptions import chartbuilder_exceptions @@ -59,7 +60,9 @@ class BaseChartBuilderTestCase(testtools.TestCase): """ chart_stream = """ - chart: + metadata: + name: test + data: chart_name: mariadb release: mariadb namespace: openstack @@ -87,7 +90,9 @@ class BaseChartBuilderTestCase(testtools.TestCase): """ dependency_chart_stream = """ - chart: + metadata: + name: dep + data: chart_name: keystone release: keystone namespace: undercloud @@ -120,6 +125,16 @@ class BaseChartBuilderTestCase(testtools.TestCase): self.addCleanup(shutil.rmtree, subdir) return subdir + def _get_test_chart(self, chart_dir): + return { + 'metadata': { + 'name': 'test' + }, + const.KEYWORD_DATA: { + 'source_dir': (chart_dir.path, '') + } + } + class ChartBuilderTestCase(BaseChartBuilderTestCase): @@ -131,8 +146,7 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) - test_chart = {'source_dir': (chart_dir.path, '')} - chartbuilder = ChartBuilder(test_chart) + chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) # Validate response type is :class:`hapi.chart.metadata_pb2.Metadata` resp = chartbuilder.get_metadata() @@ -142,8 +156,7 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) - test_chart = {'source_dir': (chart_dir.path, '')} - chartbuilder = ChartBuilder(test_chart) + chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) self.assertRaises(chartbuilder_exceptions.MetadataLoadException, chartbuilder.get_metadata) @@ -168,8 +181,7 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): for filename in ['template%d' % x for x in range(3)]: self._write_temporary_file_contents(templates_subdir, filename, "") - test_chart = {'source_dir': (chart_dir.path, '')} - chartbuilder = ChartBuilder(test_chart) + chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) expected_files = ( '[type_url: "%s"\n, type_url: "%s"\n]' % ('./bar', './foo')) @@ -185,8 +197,7 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): self._write_temporary_file_contents( chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1") - test_chart = {'source_dir': (chart_dir.path, '')} - chartbuilder = ChartBuilder(test_chart) + chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) chartbuilder.get_files() def test_get_basic_helm_chart(self): @@ -197,8 +208,8 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): self.addCleanup(shutil.rmtree, chart_dir.path) self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) - ch = yaml.safe_load(self.chart_stream)['chart'] - ch['source_dir'] = (chart_dir.path, '') + ch = yaml.safe_load(self.chart_stream) + ch['data']['source_dir'] = (chart_dir.path, '') test_chart = ch chartbuilder = ChartBuilder(test_chart) @@ -228,8 +239,8 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): self._write_temporary_file_contents(chart_dir.path, 'values.yaml', self.chart_value) - ch = yaml.safe_load(self.chart_stream)['chart'] - ch['source_dir'] = (chart_dir.path, '') + ch = yaml.safe_load(self.chart_stream) + ch['data']['source_dir'] = (chart_dir.path, '') test_chart = ch chartbuilder = ChartBuilder(test_chart) @@ -257,8 +268,8 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): 'nested') self._write_temporary_file_contents(nested_dir, 'nested0', "random") - ch = yaml.safe_load(self.chart_stream)['chart'] - ch['source_dir'] = (chart_dir.path, '') + ch = yaml.safe_load(self.chart_stream) + ch['data']['source_dir'] = (chart_dir.path, '') test_chart = ch chartbuilder = ChartBuilder(test_chart) @@ -313,8 +324,8 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): # Files to **include** within charts/ subdirectory. self._write_temporary_file_contents(charts_subdir, '.prov', "xyzzy") - ch = yaml.safe_load(self.chart_stream)['chart'] - ch['source_dir'] = (chart_dir.path, '') + ch = yaml.safe_load(self.chart_stream) + ch['data']['source_dir'] = (chart_dir.path, '') test_chart = ch chartbuilder = ChartBuilder(test_chart) @@ -340,8 +351,8 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): self.addCleanup(shutil.rmtree, chart_dir.path) self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) - ch = yaml.safe_load(self.chart_stream)['chart'] - ch['source_dir'] = (chart_dir.path, '') + ch = yaml.safe_load(self.chart_stream) + ch['data']['source_dir'] = (chart_dir.path, '') # Dependency chart directory and files. dep_chart_dir = self.useFixture(fixtures.TempDir()) @@ -349,11 +360,11 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): self._write_temporary_file_contents(dep_chart_dir.path, 'Chart.yaml', self.dependency_chart_yaml) dep_ch = yaml.safe_load(self.dependency_chart_stream) - dep_ch['chart']['source_dir'] = (dep_chart_dir.path, '') + dep_ch['data']['source_dir'] = (dep_chart_dir.path, '') main_chart = ch dependency_chart = dep_ch - main_chart['dependencies'] = [dependency_chart] + main_chart['data']['dependencies'] = [dependency_chart] chartbuilder = ChartBuilder(main_chart) helm_chart = chartbuilder.get_helm_chart() @@ -409,8 +420,8 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): self.addCleanup(shutil.rmtree, chart_dir.path) self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) - ch = yaml.safe_load(self.chart_stream)['chart'] - ch['source_dir'] = (chart_dir.path, '') + ch = yaml.safe_load(self.chart_stream) + ch['data']['source_dir'] = (chart_dir.path, '') test_chart = ch chartbuilder = ChartBuilder(test_chart) @@ -424,10 +435,10 @@ class ChartBuilderTestCase(BaseChartBuilderTestCase): self._write_temporary_file_contents(dep_chart_dir.path, 'Chart.yaml', self.dependency_chart_yaml) dep_ch = yaml.safe_load(self.dependency_chart_stream) - dep_ch['chart']['source_dir'] = (dep_chart_dir.path, '') + dep_ch['data']['source_dir'] = (dep_chart_dir.path, '') dependency_chart = dep_ch - test_chart['dependencies'] = [dependency_chart] + test_chart['data']['dependencies'] = [dependency_chart] chartbuilder = ChartBuilder(test_chart) re = inspect.cleandoc(""" @@ -457,8 +468,7 @@ class ChartBuilderNegativeTestCase(BaseChartBuilderTestCase): self._write_temporary_file_contents( chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1") - test_chart = {'source_dir': (chart_dir.path, '')} - chartbuilder = ChartBuilder(test_chart) + chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) # Confirm it failed for both encodings. error_re = (r'.*A str exception occurred while trying to read file:' @@ -477,8 +487,7 @@ class ChartBuilderNegativeTestCase(BaseChartBuilderTestCase): self._write_temporary_file_contents( chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1") - test_chart = {'source_dir': (chart_dir.path, '')} - chartbuilder = ChartBuilder(test_chart) + chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) side_effects = [self.exc_to_raise, "", ""] with mock.patch("builtins.open", mock.mock_open(read_data="")) \ diff --git a/armada/tests/unit/handlers/test_manifest.py b/armada/tests/unit/handlers/test_manifest.py index 8293bdc9..be65ab72 100644 --- a/armada/tests/unit/handlers/test_manifest.py +++ b/armada/tests/unit/handlers/test_manifest.py @@ -18,9 +18,9 @@ import yaml import testtools -from armada import const from armada import exceptions from armada.handlers import manifest +from armada.handlers import schema from armada.utils import validate @@ -111,7 +111,7 @@ class ManifestTestCase(testtools.TestCase): self.documents, target_manifest='armada-manifest') obtained_manifest = armada_manifest.get_manifest() self.assertIsInstance(obtained_manifest, dict) - self.assertEqual(obtained_manifest['armada'], + self.assertEqual(obtained_manifest['data'], armada_manifest.manifest['data']) def test_find_documents(self): @@ -194,19 +194,15 @@ class ManifestTestCase(testtools.TestCase): # the first chart group in the Armada manifest keystone_infra_services_chart_group = armada_manifest. \ find_chart_group_document('keystone-infra-services') - keystone_infra_services_chart_group_data = \ - keystone_infra_services_chart_group.get('data') - self.assertEqual(keystone_infra_services_chart_group_data, + self.assertEqual(keystone_infra_services_chart_group, built_armada_manifest['data']['chart_groups'][0]) # the first chart group in the Armada manifest openstack_keystone_chart_group = armada_manifest. \ find_chart_group_document('openstack-keystone') - openstack_keystone_chart_group_data = \ - openstack_keystone_chart_group.get('data') - self.assertEqual(openstack_keystone_chart_group_data, + self.assertEqual(openstack_keystone_chart_group, built_armada_manifest['data']['chart_groups'][1]) def test_verify_build_chart_group_deps(self): @@ -218,7 +214,7 @@ class ManifestTestCase(testtools.TestCase): build_chart_group(chart_group) openstack_keystone_chart_group_deps_dep_added = \ openstack_keystone_chart_group_deps[ - 'data']['chart_group'][0]['chart']['dependencies'] + 'data']['chart_group'][0]['data']['dependencies'] # keystone chart dependencies keystone_chart = armada_manifest.find_chart_document('keystone') @@ -237,7 +233,7 @@ class ManifestTestCase(testtools.TestCase): build_chart_group(chart_group) keystone_infra_services_dep_added = \ openstack_keystone_chart_group_deps[ - 'data']['chart_group'][0]['chart']['dependencies'] + 'data']['chart_group'][0]['data']['dependencies'] # building mariadb chart dependencies mariadb_chart = armada_manifest.find_chart_document('mariadb') @@ -274,9 +270,7 @@ class ManifestTestCase(testtools.TestCase): # helm-toolkit dependency, the basis for comparison of d # ependencies in other charts - expected_helm_toolkit_dependency = { - 'chart': helm_toolkit_chart.get('data') - } + expected_helm_toolkit_dependency = helm_toolkit_chart # keystone chart dependencies keystone_chart = armada_manifest.find_chart_document('keystone') @@ -366,42 +360,39 @@ class ManifestNegativeTestCase(testtools.TestCase): documents, target_manifest='armada-manifest') + def _assert_missing_documents_raises(self, documents): + error_re = ('.*Documents must include at least one of each of .* and ' + 'only one .*') + self.assertRaisesRegexp(exceptions.ManifestException, error_re, + manifest.Manifest, documents) + def test_get_documents_missing_manifest(self): # Validates exceptions.ManifestException is thrown if no manifest is # found. Manifest is last document in sample YAML. - error_re = ('Documents must be a list of documents with at least one ' - 'of each of the following schemas: .*') - self.assertRaisesRegexp(exceptions.ManifestException, error_re, - manifest.Manifest, self.documents[:-1]) + self._assert_missing_documents_raises(self.documents[:-1]) def test_get_documents_missing_charts(self): # Validates exceptions.ManifestException is thrown if no chart is # found. Charts are first 5 documents in sample YAML. - error_re = ('Documents must be a list of documents with at least one ' - 'of each of the following schemas: .*') - self.assertRaisesRegexp(exceptions.ManifestException, error_re, - manifest.Manifest, self.documents[5:]) + self._assert_missing_documents_raises(self.documents[5:]) def test_get_documents_missing_chart_groups(self): # Validates exceptions.ManifestException is thrown if no chart is # found. ChartGroups are 5-6 documents in sample YAML. documents = self.documents[:4] + [self.documents[-1]] - error_re = ('Documents must be a list of documents with at least one ' - 'of each of the following schemas: .*') - self.assertRaisesRegexp(exceptions.ManifestException, error_re, - manifest.Manifest, documents) + self._assert_missing_documents_raises(documents) def test_find_chart_document_negative(self): armada_manifest = manifest.Manifest(self.documents) - error_re = r'Could not build %s named "%s"' % (const.DOCUMENT_CHART, - 'invalid') + error_re = r'.*Could not find %s named "%s"' % (schema.TYPE_CHART, + 'invalid') self.assertRaisesRegexp(exceptions.BuildChartException, error_re, armada_manifest.find_chart_document, 'invalid') def test_find_group_document_negative(self): armada_manifest = manifest.Manifest(self.documents) - error_re = r'Could not build %s named "%s"' % (const.DOCUMENT_GROUP, - 'invalid') + error_re = r'.*Could not find %s named "%s"' % (schema.TYPE_CHARTGROUP, + 'invalid') self.assertRaisesRegexp(exceptions.BuildChartGroupException, error_re, armada_manifest.find_chart_group_document, 'invalid') diff --git a/armada/tests/unit/handlers/test_override.py b/armada/tests/unit/handlers/test_override.py index ded286a9..91e75b0d 100644 --- a/armada/tests/unit/handlers/test_override.py +++ b/armada/tests/unit/handlers/test_override.py @@ -20,8 +20,8 @@ import yaml import testtools from armada.handlers.override import Override +from armada.handlers import schema from armada.exceptions import override_exceptions -from armada import const class OverrideTestCase(testtools.TestCase): @@ -117,13 +117,13 @@ class OverrideTestCase(testtools.TestCase): documents = list(yaml.safe_load_all(f.read())) ovr = Override(documents) test_group = ovr.find_document_type('chart_group') - self.assertEqual(test_group, const.DOCUMENT_GROUP) + self.assertEqual(test_group, schema.TYPE_CHARTGROUP) test_chart = ovr.find_document_type('chart') - self.assertEqual(test_chart, const.DOCUMENT_CHART) + self.assertEqual(test_chart, schema.TYPE_CHART) test_manifest = ovr.find_document_type('manifest') - self.assertEqual(test_manifest, const.DOCUMENT_MANIFEST) + self.assertEqual(test_manifest, schema.TYPE_MANIFEST) def test_update_chart_document_valid(self): with open(self.base_manifest) as f: @@ -138,7 +138,7 @@ class OverrideTestCase(testtools.TestCase): ovr = Override(documents) # update with document values with the modified ones - ovr.update_chart_document(documents_modified[0]) + ovr.update_document(documents_modified[0]) # after the update, both documents are equal self.assertEqual(ovr.documents[0]['data']['chart_name'], @@ -148,7 +148,7 @@ class OverrideTestCase(testtools.TestCase): # Case 2: Checking if dictionaries get updated documents_modified[0]['data']['values'] = {'foo': 'bar'} - ovr.update_chart_document(documents_modified[0]) + ovr.update_document(documents_modified[0]) # after the update, both documents are equal self.assertEqual(ovr.documents[0]['data']['values'], @@ -158,7 +158,7 @@ class OverrideTestCase(testtools.TestCase): # Case 3: Checking if lists get updated documents_modified[0]['data']['dependencies'] = ['foo', 'bar'] - ovr.update_chart_document(documents_modified[0]) + ovr.update_document(documents_modified[0]) # after the update, both documents are equal self.assertEqual(['foo', 'bar'], @@ -179,7 +179,7 @@ class OverrideTestCase(testtools.TestCase): ovr = Override(documents) # update with document values with the modified ones - ovr.update_chart_document(documents_modified[0]) + ovr.update_document(documents_modified[0]) self.assertIn('chart_name', ovr.documents[0]['data']) self.assertNotEqual(ovr.documents[0], documents_modified[0]) @@ -195,7 +195,7 @@ class OverrideTestCase(testtools.TestCase): ovr = Override(documents) # update with document values with the modified ones - ovr.update_chart_group_document(documents_modified[1]) + ovr.update_document(documents_modified[1]) # after the update, both documents are equal self.assertEqual(ovr.documents[1]['data']['sequenced'], @@ -214,7 +214,7 @@ class OverrideTestCase(testtools.TestCase): ovr = Override(documents) # update with document values with the modified ones - ovr.update_chart_group_document(documents_modified[1]) + ovr.update_document(documents_modified[1]) self.assertIn('sequenced', ovr.documents[1]['data']) self.assertNotEqual(ovr.documents[1], documents_modified[1]) @@ -230,7 +230,7 @@ class OverrideTestCase(testtools.TestCase): ovr = Override(documents) # update with document values with the modified ones - ovr.update_armada_manifest(documents_modified[2]) + ovr.update_document(documents_modified[2]) # after the update, both documents are equal self.assertEqual(ovr.documents[2]['data']['release_prefix'], @@ -249,7 +249,7 @@ class OverrideTestCase(testtools.TestCase): ovr = Override(documents) # update with document values from base_manifest - ovr.update_armada_manifest(documents_modified[2]) + ovr.update_document(documents_modified[2]) self.assertIn('release_prefix', ovr.documents[2]['data']) self.assertNotEqual(ovr.documents[2], documents_modified[2]) @@ -265,7 +265,7 @@ class OverrideTestCase(testtools.TestCase): documents = list(yaml.safe_load_all(f.read())) doc_path = ['chart', 'blog-1'] ovr = Override(documents) - ovr.update_document(merging_values) + ovr.update_documents(merging_values) ovr_doc = ovr.find_manifest_document(doc_path) expect_doc = list(yaml.load_all(e.read()))[0] diff --git a/armada/tests/unit/utils/schema.py b/armada/tests/unit/utils/schema.py new file mode 100644 index 00000000..704301dc --- /dev/null +++ b/armada/tests/unit/utils/schema.py @@ -0,0 +1,38 @@ +# Copyright 2019 The Armada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from armada.utils import schema + + +class SchemaTestCase(unittest.TestCase): + + def test_validate_load_schemas(self): + expected_schemas = [ + 'armada/Chart/v1', 'armada/ChartGroup/v1', 'armada/Manifest/v1' + 'armada/Chart/v2', 'armada/ChartGroup/v2', 'armada/Manifest/v2' + ] + for expected_schema in expected_schemas: + self.assertIn(expected_schema, schema._SCHEMAS) + + def test_validate_load_duplicate_schemas_expect_runtime_error(self): + """Validate that calling ``_load_schemas`` results in a + ``RuntimeError`` being thrown, because the call is made during module + import, and importing the schemas again in manually results in + duplicates. + """ + with self.assertRaisesRegexp(RuntimeError, + 'Duplicate schema specified for: .*'): + schema._load_schemas() diff --git a/armada/tests/unit/utils/test_validate.py b/armada/tests/unit/utils/test_validate.py index 0572399d..1725f1c4 100644 --- a/armada/tests/unit/utils/test_validate.py +++ b/armada/tests/unit/utils/test_validate.py @@ -111,13 +111,6 @@ class ValidateOwnExamplesTestCase(BaseValidateTest): class ValidateTestCase(BaseValidateTest): - def test_validate_load_schemas(self): - expected_schemas = [ - 'armada/Chart/v1', 'armada/ChartGroup/v1', 'armada/Manifest/v1' - ] - for expected_schema in expected_schemas: - self.assertIn(expected_schema, validate.SCHEMAS) - def test_validate_armada_yaml_passes(self): template = '{}/resources/valid_armada_document.yaml'.format( self.basepath) @@ -223,16 +216,6 @@ data: class ValidateNegativeTestCase(BaseValidateTest): - def test_validate_load_duplicate_schemas_expect_runtime_error(self): - """Validate that calling ``validate._load_schemas`` results in a - ``RuntimeError`` being thrown, because the call is made during module - import, and importing the schemas again in manually results in - duplicates. - """ - with self.assertRaisesRegexp(RuntimeError, - 'Duplicate schema specified for: .*'): - validate._load_schemas() - def test_validate_no_dictionary_expect_type_error(self): expected_error = 'The provided input "invalid" must be a dictionary.' self.assertRaisesRegexp(TypeError, expected_error, diff --git a/armada/utils/validate.py b/armada/utils/validate.py index 1f93e1af..eb321fcc 100644 --- a/armada/utils/validate.py +++ b/armada/utils/validate.py @@ -13,44 +13,18 @@ # limitations under the License. import jsonschema -import os -import pkg_resources import requests import traceback -import yaml from oslo_log import log as logging -from armada.const import KEYWORD_GROUPS, KEYWORD_CHARTS, KEYWORD_RELEASE +from armada import const +from armada.handlers import schema as sch from armada.handlers.manifest import Manifest from armada.exceptions.manifest_exceptions import ManifestException from armada.utils.validation_message import ValidationMessage LOG = logging.getLogger(__name__) -# Creates a mapping between ``metadata.name``: ``data`` where the -# ``metadata.name`` is the ``schema`` of a manifest and the ``data`` is the -# JSON schema to be used to validate the manifest in question. -SCHEMAS = {} - - -def _get_schema_dir(): - return pkg_resources.resource_filename('armada', 'schemas') - - -def _load_schemas(): - """Populates ``SCHEMAS`` with the schemas defined in package - ``armada.schemas``. - - """ - schema_dir = _get_schema_dir() - for schema_file in os.listdir(schema_dir): - with open(os.path.join(schema_dir, schema_file)) as f: - for schema in yaml.safe_load_all(f): - name = schema['metadata']['name'] - if name in SCHEMAS: - raise RuntimeError( - 'Duplicate schema specified for: %s.' % name) - SCHEMAS[name] = schema['data'] def _validate_armada_manifest(manifest): @@ -71,7 +45,7 @@ def _validate_armada_manifest(manifest): details = [] try: - armada_object = manifest.get_manifest().get('armada') + manifest.get_manifest().get(const.KEYWORD_DATA) except ManifestException as me: vmsg = ValidationMessage( message=str(me), error=True, name='ARM001', level='Error') @@ -80,27 +54,6 @@ def _validate_armada_manifest(manifest): details.append(vmsg.get_output()) return False, details - groups = armada_object.get(KEYWORD_GROUPS) - - if not isinstance(groups, list): - message = '{} entry is of wrong type: {} (expected: {})'.format( - KEYWORD_GROUPS, type(groups), 'list') - vmsg = ValidationMessage( - message=message, error=True, name='ARM101', level='Error') - LOG.info('ValidationMessage: %s', vmsg.get_output_json()) - details.append(vmsg.get_output()) - - for group in groups: - for chart in group.get(KEYWORD_CHARTS): - chart_obj = chart.get('chart') - if KEYWORD_RELEASE not in chart_obj: - message = 'Could not find {} keyword in {}'.format( - KEYWORD_RELEASE, chart_obj.get('release')) - vmsg = ValidationMessage( - message=message, error=True, name='ARM102', level='Error') - LOG.info('ValidationMessage: %s', vmsg.get_output_json()) - details.append(vmsg.get_output()) - if len([x for x in details if x.get('error', False)]) > 0: return False, details @@ -117,13 +70,16 @@ def validate_armada_manifests(documents): all_valid = True for document in documents: - if document.get('schema', '') == 'armada/Manifest/v1': - target = document.get('metadata').get('name') - # TODO(MarshM) explore: why does this pass 'documents'? - manifest = Manifest(documents, target_manifest=target) - is_valid, details = _validate_armada_manifest(manifest) - all_valid = all_valid and is_valid - messages.extend(details) + doc_schema = document.get('schema') + if doc_schema: + schema_info = sch.get_schema_info(doc_schema) + if schema_info and schema_info.type == sch.TYPE_MANIFEST: + target = document.get('metadata').get('name') + # TODO(MarshM) explore: why does this pass 'documents'? + manifest = Manifest(documents, target_manifest=target) + is_valid, details = _validate_armada_manifest(manifest) + all_valid = all_valid and is_valid + messages.extend(details) return all_valid, messages @@ -151,9 +107,10 @@ def validate_armada_document(document): details = [] LOG.debug('Validating document [%s] %s', schema, document_name) - if schema in SCHEMAS: + schema_info = sch.get_schema_info(schema) + if schema_info: try: - validator = jsonschema.Draft4Validator(SCHEMAS[schema]) + validator = jsonschema.Draft4Validator(schema_info.data) for error in validator.iter_errors(document.get('data')): error_message = "Invalid document [%s] %s: %s." % \ (schema, document_name, error.message) @@ -222,7 +179,3 @@ def validate_manifest_url(value): return (requests.get(value).status_code == 200) except requests.exceptions.RequestException: return False - - -# Fill the cache. -_load_schemas() diff --git a/doc/source/operations/documents/index.rst b/doc/source/operations/documents/index.rst new file mode 100644 index 00000000..7125d84d --- /dev/null +++ b/doc/source/operations/documents/index.rst @@ -0,0 +1,26 @@ +.. + Copyright 2019 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Document Authoring Guide +======================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + v1/index + v2/index + migration-v1-v2 diff --git a/doc/source/operations/documents/migration-v1-v2.rst b/doc/source/operations/documents/migration-v1-v2.rst new file mode 100644 index 00000000..51fee18d --- /dev/null +++ b/doc/source/operations/documents/migration-v1-v2.rst @@ -0,0 +1,70 @@ +.. + Copyright 2019 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +v1-v2 Migration +=============== + +The following migrations must be done when moving from :ref:`v1 ` to :ref:`v2 ` docs. + +Chart +----- + ++--------------------------------+------------------------------------------------------------+ +| change | migration | ++================================+============================================================+ +| ``chart_name`` removed | Remove. It was redundant with ``metadata.name`` while at | +| | the same time not guaranteeing uniqueness. Log messages now| +| | reference ``metadata.name`` for improved grep-ability. | ++--------------------------------+------------------------------------------------------------+ +| ``test`` as a boolean removed | :ref:`test ` must now be an object. | ++--------------------------------+------------------------------------------------------------+ +| ``timeout`` removed | Use ``wait.timeout`` instead. | ++--------------------------------+------------------------------------------------------------+ +| ``install`` removed | Remove. Previously unused. | ++--------------------------------+------------------------------------------------------------+ +| ``upgrade.post`` removed | Remove. | ++--------------------------------+------------------------------------------------------------+ +| ``upgrade.pre.update`` removed | Remove. | ++--------------------------------+------------------------------------------------------------+ +| ``upgrade.pre.create`` removed | Remove. | ++--------------------------------+------------------------------------------------------------+ +| ``upgrade.pre.delete[*].name`` | Remove. | +| removed | | ++--------------------------------+------------------------------------------------------------+ +| ``upgrade.pre.delete[*]`` | If you have an item in ``upgrade.pre.delete`` and | +| with ``type: job`` no longer | ``type: job`` and you also want to delete cronjobs, add | +| deletes cronjobs | another item with ``type: cronjob`` and same labels. | ++--------------------------------+------------------------------------------------------------+ +| ``dependencies``, | Remove as desired. | +| ``upgrade.no_hooks``, | | +| ``source.subpath`` | | +| now optional | | ++--------------------------------+------------------------------------------------------------+ + +ChartGroup +---------- + ++--------------------------+-----------------------------------------------------------+ +| change | migration | ++==========================+===========================================================+ +| ``test_charts`` removed | Use the Chart schema's :ref:`test.enabled ` | +| | instead. | ++--------------------------+-----------------------------------------------------------+ + +Manifest +-------- + +No changes. diff --git a/doc/source/operations/guide-build-armada-yaml.rst b/doc/source/operations/documents/v1/document-authoring.rst similarity index 97% rename from doc/source/operations/guide-build-armada-yaml.rst rename to doc/source/operations/documents/v1/document-authoring.rst index ae5c7ce5..7c623952 100644 --- a/doc/source/operations/guide-build-armada-yaml.rst +++ b/doc/source/operations/documents/v1/document-authoring.rst @@ -1,5 +1,23 @@ -Armada - Making Your First Armada Manifest -========================================== +.. + Copyright 2019 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _document_authoring_v1: + +v1 Authoring +============ armada/Manifest/v1 ------------------ @@ -191,9 +209,6 @@ Run helm tests on the chart after install/upgrade. deprecated and will be removed. The ``cleanup`` option below is set to true in this case for backward compatibility. -.. _test_options: - - Test Options ^^^^^^^^^^^^ diff --git a/doc/source/operations/documents/v1/index.rst b/doc/source/operations/documents/v1/index.rst new file mode 100644 index 00000000..0a68c4eb --- /dev/null +++ b/doc/source/operations/documents/v1/index.rst @@ -0,0 +1,25 @@ +.. + Copyright 2019 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +v1 +== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + document-authoring + schemas diff --git a/doc/source/operations/documents.rst b/doc/source/operations/documents/v1/schemas.rst similarity index 88% rename from doc/source/operations/documents.rst rename to doc/source/operations/documents/v1/schemas.rst index d660171d..7128feb2 100644 --- a/doc/source/operations/documents.rst +++ b/doc/source/operations/documents/v1/schemas.rst @@ -14,15 +14,11 @@ License for the specific language governing permissions and limitations under the License. -.. _armada-documents: +v1 Schemas +========== -Armada Documents -================ - -Below are the schemas Armada uses to validate :ref:`Charts`, -:ref:`Chart Groups`, and :ref:`Manifests`. - -.. _Charts: +Below are the schemas Armada uses to validate Charts, Chart Groups, and +Manifests. Charts ------ @@ -32,16 +28,12 @@ comparable to a Helm chart. Charts consist of all the labels, dependencies, install and upgrade information, hooks and additional information needed to convey to Tiller. -.. _Chart Groups: - Chart Groups ------------ A ``Chart Group`` consists of a list of charts. ``Chart Group`` documents are useful for managing a group of ``Chart`` documents together. -.. _Manifests: - Manifests --------- @@ -76,7 +68,7 @@ Schemas ``metadata.name`` are validated. .. literalinclude:: - ../../../armada/schemas/armada-chart-schema.yaml + ../../../../../armada/schemas/armada-chart-schema-v1.yaml :language: yaml :lines: 15- :caption: Schema for ``armada/Chart/v1`` documents. @@ -90,7 +82,7 @@ Schemas ``metadata.name`` are validated. .. literalinclude:: - ../../../armada/schemas/armada-chartgroup-schema.yaml + ../../../../../armada/schemas/armada-chartgroup-schema-v1.yaml :language: yaml :lines: 15- :caption: Schema for ``armada/ChartGroup/v1`` documents. @@ -104,7 +96,7 @@ Schemas ``metadata.name`` are validated. .. literalinclude:: - ../../../armada/schemas/armada-manifest-schema.yaml + ../../../../../armada/schemas/armada-manifest-schema-v1.yaml :language: yaml :lines: 15- :caption: Schema for ``armada/Manifest/v1`` documents. @@ -112,8 +104,6 @@ Schemas This schema is used to sanity-check all ``Manifest`` documents that are passed to Armada. -.. _authoring-guidelines: - Authoring Guidelines -------------------- diff --git a/doc/source/operations/documents/v2/document-authoring.rst b/doc/source/operations/documents/v2/document-authoring.rst new file mode 100644 index 00000000..5880a436 --- /dev/null +++ b/doc/source/operations/documents/v2/document-authoring.rst @@ -0,0 +1,469 @@ +.. + Copyright 2019 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _document_authoring_v2: + +v2 Authoring +============ + +.. DANGER:: + + EXPERIMENTAL: `v2` docs are still experimental and WILL have breaking changes + before they are finalized. + +armada/Manifest/v2 +------------------ + ++---------------------+--------+-------------------------+ +| keyword | type | action | ++=====================+========+=========================+ +| ``release_prefix`` | string | appends to the | +| | | front of all | +| | | charts | +| | | released | +| | | by the | +| | | manifest in | +| | | order to | +| | | manage releases | +| | | throughout their | +| | | lifecycle | ++---------------------+--------+-------------------------+ +| ``chart_groups`` | array | A list of the | +| | | ``metadata.name`` of | +| | | each ``ChartGroup`` to | +| | | deploy in order. | ++---------------------+--------+-------------------------+ + +Manifest Example +^^^^^^^^^^^^^^^^ + +:: + + --- + schema: armada/Manifest/v2 + metadata: + schema: metadata/Document/v1 + name: simple-armada + data: + release_prefix: armada + chart_groups: + - chart_group + + +armada/ChartGroup/v2 +-------------------- + ++-----------------+----------+------------------------------------------------------------------------+ +| keyword | type | action | ++=================+==========+========================================================================+ +| description | string | description of chart set | ++-----------------+----------+------------------------------------------------------------------------+ +| chart_group | array | A list of the ``metadata.name`` of each ``Chart`` to deploy. | ++-----------------+----------+------------------------------------------------------------------------+ +| sequenced | bool | If ``true``, deploys each chart in sequence, else in parallel. | +| | | Default ``false``. | ++-----------------+----------+------------------------------------------------------------------------+ + +Chart Group Example +^^^^^^^^^^^^^^^^^^^ + +:: + + --- + schema: armada/ChartGroup/v2 + metadata: + schema: metadata/Document/v1 + name: blog-group + data: + description: Deploys Simple Service + chart_group: + - chart1 + - chart2 + +armada/Chart/v2 +--------------- + +Chart +^^^^^ + ++-----------------+----------+---------------------------------------------------------------------------------------+ +| keyword | type | action | ++=================+==========+=======================================================================================+ +| release | string | name of the release (Armada will prepend with ``release-prefix`` during processing) | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| namespace | string | namespace of your chart | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| wait | object | See `Wait`_. | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| protected | object | do not delete FAILED releases when encountered from previous run (provide the | +| | | 'continue_processing' bool to continue or halt execution (default: halt)) | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| test | object | See Test_. | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| upgrade | object | upgrade the chart managed by the armada yaml | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| delete | object | See Delete_. | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| values | object | (optional) override any default values in the charts | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| source | object | provide a path to a ``git repo``, ``local dir``, or ``tarball url`` chart | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| dependencies | object | (optional) reference any chart dependencies before install | ++-----------------+----------+---------------------------------------------------------------------------------------+ + +Wait +^^^^ + ++-------------+----------+--------------------------------------------------------------------+ +| keyword | type | action | ++=============+==========+====================================================================+ +| timeout | int | time (in seconds) to wait for chart to deploy | ++-------------+----------+--------------------------------------------------------------------+ +| resources | array | Array of `Wait Resource`_ to wait on, with ``labels`` added to each| +| | | item. Defaults to pods and jobs (if any exist) matching ``labels``.| ++-------------+----------+--------------------------------------------------------------------+ +| labels | object | Base mapping of labels to wait on. They are added to any labels in | +| | | each item in the ``resources`` array. | ++-------------+----------+--------------------------------------------------------------------+ +| native | boolean | See `Wait Native`_. | ++-------------+----------+--------------------------------------------------------------------+ + +Wait Resource +^^^^^^^^^^^^^ ++-------------+----------+--------------------------------------------------------------------+ +| keyword | type | action | ++=============+==========+====================================================================+ +| type | string | k8s resource type, supports: controllers ('deployment', | +| | | 'daemonset', 'statefulset'), 'pod', 'job' | ++-------------+----------+--------------------------------------------------------------------+ +| labels | object | mapping of kubernetes resource labels | ++-------------+----------+--------------------------------------------------------------------+ +| min\_ready | int | Only for controller ``type``s. Amount of pods in a controller | +| | string | which must be ready. Can be integer or percent string e.g. ``80%``.| +| | | Default ``100%``. | ++-------------+----------+--------------------------------------------------------------------+ + +Wait Native +^^^^^^^^^^^ + +Config for the native ``helm (install|upgrade) --wait`` flag. + ++-------------+----------+--------------------------------------------------------------------+ +| keyword | type | action | ++=============+==========+====================================================================+ +| enabled | boolean | defaults to true | ++-------------+----------+--------------------------------------------------------------------+ + +.. _test_v2: + +Test +^^^^ + +Run helm tests on the chart after install/upgrade. + ++-------------+----------+--------------------------------------------------------------------+ +| keyword | type | action | ++=============+==========+====================================================================+ +| enabled | bool | whether to enable/disable helm tests for this chart (default True) | ++-------------+----------+--------------------------------------------------------------------+ +| timeout | int | time (in sec) to wait for completion of Helm tests | ++-------------+----------+--------------------------------------------------------------------+ +| options | object | See `Test Options`_. | ++-------------+----------+--------------------------------------------------------------------+ + +.. note:: + + Armada will attempt to run helm tests by default. They may be disabled by + setting the ``enabled`` key to ``False``. + +Test Options +^^^^^^^^^^^^ + +Test options to pass through directly to helm. + ++-------------+----------+---------------------------------------------------------------+ +| keyword | type | action | ++=============+==========+===============================================================+ +| cleanup | bool | cleanup test pods after test completion, defaults to false | ++-------------+----------+---------------------------------------------------------------+ + +.. note:: + + If cleanup is ``true`` this prevents being able to debug a test in the event of failure. + + Historically, the preferred way to achieve test cleanup has been to add a pre-upgrade delete + action on the test pod. + + This still works, however it is usually no longer necessary as Armada now automatically + cleans up any test pods which match the ``wait.labels`` of the chart, immediately before + running tests. Similar suggestions have been made for how ``helm test --cleanup`` itself + ought to work (https://github.com/helm/helm/issues/3279). + +Upgrade - Pre +^^^^^^^^^^^^^ + ++-------------+----------+---------------------------------------------------------------+ +| keyword | type | action | ++=============+==========+===============================================================+ +| pre | object | actions performed prior to updating a release | ++-------------+----------+---------------------------------------------------------------+ + +Upgrade - Actions +^^^^^^^^^^^^^^^^^ + ++-------------+----------+---------------------------------------------------------------+ +| keyword | type | action | ++=============+==========+===============================================================+ +| delete | array | List of `Upgrade - Actions - Delete`_. | ++-------------+----------+---------------------------------------------------------------+ + +Upgrade - Actions - Delete +^^^^^^^^^^^^^^^^^^^^^^^^^^ + ++-------------+----------+---------------------------------------------------------------+ +| keyword | type | action | ++=============+==========+===============================================================+ +| type | string | type of kubernetes resource to delete | +| | | supported types are: 'pod', 'job', 'cronjob'. | ++-------------+----------+---------------------------------------------------------------+ +| labels | object | k:v mapping of labels to select Kubernetes resources | ++-------------+----------+---------------------------------------------------------------+ + +Chart Example +^^^^^^^^^^^^^ + +:: + + --- + schema: armada/Chart/v2 + metadata: + schema: metadata/Document/v1 + name: blog-1 + data: + release: blog-1 + namespace: default + wait: + timeout: 100 + protected: + continue_processing: false + test: + enabled: true + upgrade: + pre: + delete: + - name: test-job + type: job + labels: + foo: bar + component: bar + rak1: enabled + source: + type: git + location: https://github.com/namespace/repo + reference: master + +Delete +^^^^^^ + ++-------------+----------+-----------------------------------------------------------------------------------+ +| keyword | type | action | ++=============+==========+===================================================================================+ +| timeout | integer | time (in seconds) to wait for chart to be deleted | ++-------------+----------+-----------------------------------------------------------------------------------+ + +Source +^^^^^^ + ++-------------+----------+-----------------------------------------------------------------------------------+ +| keyword | type | action | ++=============+==========+===================================================================================+ +| type | string | source to build the chart: ``git``, ``local``, or ``tar`` | ++-------------+----------+-----------------------------------------------------------------------------------+ +| location | string | ``url`` or ``path`` to the chart's parent directory | ++-------------+----------+-----------------------------------------------------------------------------------+ +| subpath | string | (optional) relative path to target chart from parent (``.`` if not specified) | ++-------------+----------+-----------------------------------------------------------------------------------+ +| reference | string | (optional) branch, commit, or reference in the repo (``master`` if not specified) | ++-------------+----------+-----------------------------------------------------------------------------------+ + +Source Example +^^^^^^^^^^^^^^ + +:: + + # type git + --- + schema: armada/Chart/v2 + metadata: + schema: metadata/Document/v1 + name: blog-1 + data: + release: blog-1 + namespace: default + wait: + timeout: 100 + labels: + component: blog + source: + type: git + location: https://github.com/namespace/repo + + # type local + --- + schema: armada/Chart/v2 + metadata: + schema: metadata/Document/v1 + name: blog-1 + data: + release: blog-1 + namespace: default + wait: + timeout: 100 + source: + type: local + location: /path/to/charts + subpath: chart + reference: master + + # type tar + --- + schema: armada/Chart/v2 + metadata: + schema: metadata/Document/v1 + name: blog-1 + data: + release: blog-1 + namespace: default + wait: + timeout: 100 + source: + type: tar + location: https://localhost:8879/charts/chart-0.1.0.tgz + subpath: mariadb + +Simple Example +^^^^^^^^^^^^^^ + +:: + + --- + schema: armada/Chart/v2 + metadata: + schema: metadata/Document/v1 + name: blog-1 + data: + release: blog-1 + namespace: default + source: + type: git + location: https://github.com/namespace/repo + subpath: blog-1 + reference: new-feat + --- + schema: armada/ChartGroup/v2 + metadata: + schema: metadata/Document/v1 + name: blog-group + data: + description: Deploys Simple Service + chart_group: + - blog-1 + --- + schema: armada/Manifest/v2 + metadata: + schema: metadata/Document/v1 + name: simple-armada + data: + release_prefix: armada + chart_groups: + - blog-group + +Multichart Example +^^^^^^^^^^^^^^^^^^ + +:: + + --- + schema: armada/Chart/v2 + metadata: + schema: metadata/Document/v1 + name: blog-1 + data: + release: blog-1 + namespace: default + source: + type: git + location: https://github.com/namespace/repo + subpath: blog1 + reference: master + --- + schema: armada/Chart/v2 + metadata: + schema: metadata/Document/v1 + name: blog-2 + data: + release: blog-2 + namespace: default + source: + type: tar + location: https://github.com/namespace/repo/blog2.tgz + subpath: blog2 + --- + schema: armada/Chart/v2 + metadata: + schema: metadata/Document/v1 + name: blog-3 + data: + release: blog-3 + namespace: default + source: + type: local + location: /home/user/namespace/repo/blog3 + --- + schema: armada/ChartGroup/v2 + metadata: + schema: metadata/Document/v1 + name: blog-group-1 + data: + description: Deploys Simple Service + chart_group: + - blog-2 + --- + schema: armada/ChartGroup/v2 + metadata: + schema: metadata/Document/v1 + name: blog-group-2 + data: + description: Deploys Simple Service + chart_group: + - blog-1 + - blog-3 + --- + schema: armada/Manifest/v2 + metadata: + schema: metadata/Document/v1 + name: simple-armada + data: + release_prefix: armada + chart_groups: + - blog-group-1 + - blog-group-2 + +References +~~~~~~~~~~ + +For working examples please check the examples in our repo +`here `__ diff --git a/doc/source/operations/documents/v2/index.rst b/doc/source/operations/documents/v2/index.rst new file mode 100644 index 00000000..becb27d4 --- /dev/null +++ b/doc/source/operations/documents/v2/index.rst @@ -0,0 +1,25 @@ +.. + Copyright 2019 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +v2 (EXPERIMENTAL!) +================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + document-authoring + schemas diff --git a/doc/source/operations/documents/v2/schemas.rst b/doc/source/operations/documents/v2/schemas.rst new file mode 100644 index 00000000..f1d4cc38 --- /dev/null +++ b/doc/source/operations/documents/v2/schemas.rst @@ -0,0 +1,114 @@ +.. + Copyright 2018 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +v2 Schemas +========== + +Below are the schemas Armada uses to validate Charts, Chart Groups, and +Manifests. + +Charts +------ + +Charts consist of the smallest building blocks in Armada. A ``Chart`` is +comparable to a Helm chart. Charts consist of all the labels, dependencies, +install and upgrade information, hooks and additional information needed to +convey to Tiller. + +Chart Groups +------------ + +A ``Chart Group`` consists of a list of charts. ``Chart Group`` documents are +useful for managing a group of ``Chart`` documents together. + +Manifests +--------- + +A ``Manifest`` is the largest building block in Armada. ``Manifest`` documents +are responsible for managing collections of ``Chart Group`` documents. + +Validation Schemas +------------------ + +Introduction +^^^^^^^^^^^^ + +All schemas below are `Deckhand DataSchema`_ documents, which are essentially +JSON schemas, with additional metadata useful for Deckhand to perform +`layering`_ and `substitution`_. + +The validation schemas below are used by Armada to validate all ingested +Charts, Chart Groups, and Manifests. Use the schemas below as models for +authoring Armada documents. + +.. _Deckhand DataSchema: https://airship-deckhand.readthedocs.io/en/latest/document-types.html?highlight=dataschema#dataschema +.. _Helm charts: https://docs.helm.sh/developing_charts/ +.. _layering: https://airship-deckhand.readthedocs.io/en/latest/layering.html +.. _substitution: https://airship-deckhand.readthedocs.io/en/latest/substitution.html + +Schemas +^^^^^^^ + +* ``Chart`` schema. + + JSON schema against which all documents with ``armada/Chart/v2`` + ``metadata.name`` are validated. + + .. literalinclude:: + ../../../../../armada/schemas/armada-chart-schema-v2.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``armada/Chart/v2`` documents. + + This schema is used to sanity-check all ``Chart`` documents that are passed + to Armada. + +* ``Chart Group`` schema. + + JSON schema against which all documents with ``armada/Chart/v2`` + ``metadata.name`` are validated. + + .. literalinclude:: + ../../../../../armada/schemas/armada-chartgroup-schema-v2.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``armada/ChartGroup/v2`` documents. + + This schema is used to sanity-check all ``Chart Group`` documents that are + passed to Armada. + +* ``Manifest`` schema. + + JSON schema against which all documents with ``armada/Manifest/v2`` + ``metadata.name`` are validated. + + .. literalinclude:: + ../../../../../armada/schemas/armada-manifest-schema-v2.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``armada/Manifest/v2`` documents. + + This schema is used to sanity-check all ``Manifest`` documents that are passed + to Armada. + +Authoring Guidelines +-------------------- + +All Armada documents must use the ``deckhand/DataSchema/v1`` schema. + +.. todo:: + + Expand on this section. diff --git a/doc/source/operations/guide-use-armada.rst b/doc/source/operations/guide-use-armada.rst index 567a67f5..3f479b4a 100644 --- a/doc/source/operations/guide-use-armada.rst +++ b/doc/source/operations/guide-use-armada.rst @@ -10,7 +10,9 @@ Kubernetes Cluster `Tiller Service `_ -`Armada.yaml `_ +.. todo:: point this to v2 docs once they're stable + +:ref:`Armada documents ` .. note:: diff --git a/doc/source/operations/index.rst b/doc/source/operations/index.rst index 62a5118e..590f6250 100644 --- a/doc/source/operations/index.rst +++ b/doc/source/operations/index.rst @@ -10,11 +10,10 @@ Operations Guide :maxdepth: 2 :caption: Contents: - guide-build-armada-yaml + documents/index guide-configure guide-troubleshooting guide-use-armada - documents exceptions/index guide-helm-plugin sampleconf