diff --git a/tools/pegleg.sh b/tools/pegleg.sh index df01c741..21819fd1 100755 --- a/tools/pegleg.sh +++ b/tools/pegleg.sh @@ -2,6 +2,10 @@ set -e +realpath() { + [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" +} + SCRIPT_DIR=$(realpath "$(dirname "${0}")") SOURCE_DIR=${SCRIPT_DIR}/pegleg if [ -d "$PWD/global" ]; then diff --git a/tools/pegleg/pegleg/cli.py b/tools/pegleg/pegleg/cli.py index 6fe31833..3bb79259 100644 --- a/tools/pegleg/pegleg/cli.py +++ b/tools/pegleg/pegleg/cli.py @@ -32,7 +32,6 @@ def main(ctx, *, verbose): def site(): pass - @site.command(help='Output complete config for one site') @click.option( '-o', @@ -45,7 +44,6 @@ def site(): def collect(*, output_stream, site_name): engine.site.collect(site_name, output_stream) - @site.command(help='Find sites impacted by changed files') @click.option( '-i', @@ -89,6 +87,18 @@ def show(*, output_stream, site_name): engine.site.show(site_name, output_stream) +@site.command('render', help='Render a site through the deckhand engine') +@click.option( + '-o', + '--output', + 'output_stream', + type=click.File(mode='w'), + default=sys.stdout, + help='Where to output') +@click.argument('site_name') +def render(*, output_stream, site_name): + engine.site.render(site_name, output_stream) + def _validate_revision_callback(_ctx, _param, value): if value is not None and value.startswith('v'): return value @@ -114,6 +124,14 @@ SITE_TYPE_OPTION = click.option( required=True, help='Site type to use ("large", "medium", "cicd", "labs", etc.') +LINT_OPTION = click.option( + '-f', + '--fail-on-missing-sub-src', + required=False, + type=click.BOOL, + default=True, + help="Raise deckhand exception on missing substition sources. Defaults to True.") + @stub.command('global', help='Add global structure for a new revision') @RELEASE_OPTION @@ -136,6 +154,7 @@ def site_type(*, revision, site_type): engine.stub.site_type(revision, site_type) +@LINT_OPTION @main.command(help='Sanity checks for repository content') -def lint(): - engine.lint.full() +def lint(*, fail_on_missing_sub_src): + engine.lint.full(fail_on_missing_sub_src) diff --git a/tools/pegleg/pegleg/engine/lint.py b/tools/pegleg/pegleg/engine/lint.py index 361b76ba..778bbd40 100644 --- a/tools/pegleg/pegleg/engine/lint.py +++ b/tools/pegleg/pegleg/engine/lint.py @@ -17,10 +17,11 @@ DECKHAND_SCHEMAS = { } -def full(): +def full(fail_on_missing_sub_src=False): errors = [] errors.extend(_verify_no_unexpected_files()) errors.extend(_verify_file_contents()) + errors.extend(_verify_deckhand_render(fail_on_missing_sub_src)) if errors: raise click.ClickException('\n'.join(['Linting failed:'] + errors)) @@ -122,6 +123,20 @@ def _verify_document(document, schemas, filename): % (filename, name)) return errors +def _verify_deckhand_render(fail_on_missing_sub_src=False): + + documents = [] + + for filename in util.files.all(): + with open(filename) as f: + documents.extend(list(yaml.safe_load_all(f))) + + rendered_documents, errors = util.deckhand.deckhand_render( + documents=documents, + fail_on_missing_sub_src=fail_on_missing_sub_src, + validate=True, + ) + return errors def _layer(data): if hasattr(data, 'get'): diff --git a/tools/pegleg/pegleg/engine/site.py b/tools/pegleg/pegleg/engine/site.py index 53c45dc8..27836bd9 100644 --- a/tools/pegleg/pegleg/engine/site.py +++ b/tools/pegleg/pegleg/engine/site.py @@ -2,8 +2,9 @@ from pegleg.engine import util import collections import csv import json +import yaml -__all__ = ['collect', 'impacted', 'list_', 'show'] +__all__ = ['collect', 'impacted', 'list_', 'show', 'render'] def collect(site_name, output_stream): @@ -25,6 +26,17 @@ def impacted(input_stream, output_stream): for site_name in sorted(impacted_sites): output_stream.write(site_name + '\n') +def render(site_name, output_stream): + documents = [] + for filename in util.definition.site_files(site_name): + with open(filename) as f: + documents.extend(list(yaml.safe_load_all(f))) + + rendered_documents, errors = util.deckhand.deckhand_render( + documents=documents + ) + for d in documents: + output_stream.writelines(yaml.dump(d)) def list_(output_stream): fieldnames = ['site_name', 'site_type', 'revision'] diff --git a/tools/pegleg/pegleg/engine/util/__init__.py b/tools/pegleg/pegleg/engine/util/__init__.py index 31f202dc..0597af0e 100644 --- a/tools/pegleg/pegleg/engine/util/__init__.py +++ b/tools/pegleg/pegleg/engine/util/__init__.py @@ -1,3 +1,4 @@ # flake8: noqa from . import definition from . import files +from . import deckhand diff --git a/tools/pegleg/pegleg/engine/util/deckhand.py b/tools/pegleg/pegleg/engine/util/deckhand.py new file mode 100644 index 00000000..92740d95 --- /dev/null +++ b/tools/pegleg/pegleg/engine/util/deckhand.py @@ -0,0 +1,42 @@ +from deckhand.engine import layering +from deckhand import errors as dh_errors + +def load_schemas_from_docs(documents): + ''' + Fills the cache of known schemas from the document set + ''' + + errors = [] + SCHEMA_SCHEMA = "deckhand/DataSchema/v1" + + schema_set = dict() + for document in documents: + if document.get('schema', '') == SCHEMA_SCHEMA: + name = document['metadata']['name'] + if name in schema_set: + errors.append('Duplicate schema specified for: %s' % name) + + schema_set[name] = document['data'] + + return schema_set, errors + +def deckhand_render(documents=[], fail_on_missing_sub_src=False, validate=False): + + errors = [] + rendered_documents = [] + + schemas, schema_errors = load_schemas_from_docs(documents) + errors.extend(schema_errors) + + try: + deckhand_eng = layering.DocumentLayering( + documents, + substitution_sources=documents, + fail_on_missing_sub_src=fail_on_missing_sub_src, + validate=validate) + rendered_documents = [dict(d) for d in deckhand_eng.render()] + except dh_errors.DeckhandException as e: + errors.append('An unknown Deckhand exception occurred while trying' + ' to render documents: %s' % str(e)) + + return rendered_documents, errors \ No newline at end of file diff --git a/tools/pegleg/requirements.txt b/tools/pegleg/requirements.txt index 7072f3e3..ddb540d1 100644 --- a/tools/pegleg/requirements.txt +++ b/tools/pegleg/requirements.txt @@ -1,3 +1,4 @@ click==6.7 jsonschema==2.6.0 pyyaml==3.12 +git+https://github.com/att-comdev/deckhand.git@02c6a8dc1fe1a2ff82daba18f4df573d6d20cfca#egg=deckhand