From 9113d249fff01dc89a399e13fa3ca1ff244d4be1 Mon Sep 17 00:00:00 2001 From: Nishant Kumar Date: Tue, 9 Oct 2018 22:08:20 +0000 Subject: [PATCH] CLI: Add support for uploading documents to Shipyard This PS enables Pegleg to upload documents directly to Shipyard thus ensuring that unencrypted data never gets stored in disk. The flow for this new CLI command is as follows: - Collect documents as per the provided site repository - Decrypt the collected documets(TODO) - Upload document to Shipyard: - one collection per repository will be uploaded to Shipyard Eg- pegleg site -r /opt/aic-clcp-site-manifests \ -e global=/opt/aic-clcp-manifests upload Two collections will be created in shipyard since there are two repositories provided. The name of the collections will be the name of repositories provided. - Commit the documents in shipyard buffer. Change-Id: I6275252b044ebb82d8bb2009c0bea6ebf7033bce --- doc/source/cli/cli.rst | 40 ++++ doc/source/exceptions.rst | 7 + pegleg/cli.py | 51 +++++ pegleg/engine/util/files.py | 15 ++ pegleg/engine/util/shipyard_helper.py | 181 ++++++++++++++++++ requirements.txt | 1 + tests/unit/engine/test_util_files.py | 14 ++ .../unit/engine/util/test_shipyard_helper.py | 172 +++++++++++++++++ tests/unit/test_cli.py | 17 ++ 9 files changed, 498 insertions(+) create mode 100644 pegleg/engine/util/shipyard_helper.py create mode 100644 tests/unit/engine/util/test_shipyard_helper.py diff --git a/doc/source/cli/cli.rst b/doc/source/cli/cli.rst index b58bf9b7..69e1655b 100644 --- a/doc/source/cli/cli.rst +++ b/doc/source/cli/cli.rst @@ -389,6 +389,44 @@ A more complex example involves excluding certain linting checks: lint \ -x P001 -x P002 -w P003 +Upload +------- + +Uploads documents to `Shipyard`_. + +**site_name** (Required). + +Name of the site. The ``site_name`` must match a ``site`` name in the site +repository folder structure + +**--os-=** (Required). + +Shipyard needs these options for authenticating with OpenStack Keystone. +This option can be set as environment variables or it can be passed via +the command line. + +Please reference Shipyard's `CLI documentation`_ for information related to these options. + +**--context-marker=** (Optional). + +Specifies a UUID (8-4-4-4-12 format) that will be used to correlate logs, +transactions, etc. in downstream activities triggered by this interaction. + +Usage: + +:: + + ./pegleg.sh site upload --context-marker= + +Examples +^^^^^^^^ + +:: + + ./pegleg.sh site -r -e \ + upload + + .. _command-line-repository-overrides: Secrets @@ -641,3 +679,5 @@ P003 - All repos contain expected directories. .. _Deckhand: https://airship-deckhand.readthedocs.io/en/latest/users/rendering.html .. _Deckhand Validations: https://airship-deckhand.readthedocs.io/en/latest/overview.html#validation .. _Pegleg Managed Documents: https://airship-specs.readthedocs.io/en/latest/specs/approved/pegleg-secrets.html#peglegmanageddocument +.. _Shipyard: https://github.com/openstack/airship-shipyard +.. _CLI documentation: https://airship-shipyard.readthedocs.io/en/latest/CLI.html#openstack-keystone-authorization-environment-variables diff --git a/doc/source/exceptions.rst b/doc/source/exceptions.rst index a0c5450f..8fb8577a 100644 --- a/doc/source/exceptions.rst +++ b/doc/source/exceptions.rst @@ -56,3 +56,10 @@ Git Exceptions :members: :show-inheritance: :undoc-members: + +Authentication Exceptions +------------------------- + +.. autoexception:: pegleg.engine.util.shipyard_helper.AuthValuesError + :members: + :undoc-members: diff --git a/pegleg/cli.py b/pegleg/cli.py index 84b03e4f..c7b10192 100644 --- a/pegleg/cli.py +++ b/pegleg/cli.py @@ -20,6 +20,7 @@ import click from pegleg import config from pegleg import engine +from pegleg.engine.util.shipyard_helper import ShipyardHelper LOG = logging.getLogger(__name__) @@ -325,6 +326,56 @@ def lint_site(*, fail_on_missing_sub_src, exclude_lint, warn_lint, site_name): warn_lint=warn_lint) +@site.command('upload', help='Upload documents to Shipyard') +# Keystone authentication parameters +@click.option('--os-project-domain-name', + envvar='OS_PROJECT_DOMAIN_NAME', + required=False, + default='default') +@click.option('--os-user-domain-name', + envvar='OS_USER_DOMAIN_NAME', + required=False, + default='default') +@click.option('--os-project-name', envvar='OS_PROJECT_NAME', required=False) +@click.option('--os-username', envvar='OS_USERNAME', required=False) +@click.option('--os-password', envvar='OS_PASSWORD', required=False) +@click.option( + '--os-auth-url', envvar='OS_AUTH_URL', required=False) +# Option passed to Shipyard client context +@click.option( + '--context-marker', + help='Specifies a UUID (8-4-4-4-12 format) that will be used to correlate ' + 'logs, transactions, etc. in downstream activities triggered by this ' + 'interaction ', + required=False, + type=click.UUID) +@SITE_REPOSITORY_ARGUMENT +@click.pass_context +def upload(ctx, *, os_project_domain_name, + os_user_domain_name, os_project_name, os_username, + os_password, os_auth_url, context_marker, site_name): + if not ctx.obj: + ctx.obj = {} + + # Build API parameters required by Shipyard API Client. + auth_vars = { + 'project_domain_name': os_project_domain_name, + 'user_domain_name': os_user_domain_name, + 'project_name': os_project_name, + 'username': os_username, + 'password': os_password, + 'auth_url': os_auth_url + } + + ctx.obj['API_PARAMETERS'] = { + 'auth_vars': auth_vars + } + ctx.obj['context_marker'] = str(context_marker) + ctx.obj['site_name'] = site_name + + click.echo(ShipyardHelper(ctx).upload_documents()) + + @main.group(help='Commands related to types') @MAIN_REPOSITORY_OPTION @REPOSITORY_CLONE_PATH_OPTION diff --git a/pegleg/engine/util/files.py b/pegleg/engine/util/files.py index 0ab0cae9..7a182851 100644 --- a/pegleg/engine/util/files.py +++ b/pegleg/engine/util/files.py @@ -13,11 +13,13 @@ # limitations under the License. import click +import collections import os import yaml import logging from pegleg import config +from pegleg.engine import util from pegleg.engine.util import pegleg_managed_document as md LOG = logging.getLogger(__name__) @@ -36,6 +38,7 @@ __all__ = [ 'search', 'slurp', 'check_file_save_location', + 'collect_files_by_repo', ] DIR_DEPTHS = { @@ -366,3 +369,15 @@ def check_file_save_location(save_location): raise click.ClickException( 'save_location %s already exists, ' 'but is not a directory'.format(save_location)) + + +def collect_files_by_repo(site_name): + """ Collects file by repo name in memory.""" + + collected_files_by_repo = collections.defaultdict(list) + for repo_base, filename in util.definition.site_files_by_repo( + site_name): + repo_name = os.path.normpath(repo_base).split(os.sep)[-1] + documents = util.files.read(filename) + collected_files_by_repo[repo_name].extend(documents) + return collected_files_by_repo diff --git a/pegleg/engine/util/shipyard_helper.py b/pegleg/engine/util/shipyard_helper.py new file mode 100644 index 00000000..4c58580e --- /dev/null +++ b/pegleg/engine/util/shipyard_helper.py @@ -0,0 +1,181 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import uuid + +import yaml + +from pegleg.engine.util import files +from pegleg.engine.exceptions import PeglegBaseException + +from shipyard_client.api_client.shipyard_api_client import ShipyardClient +from shipyard_client.api_client.shipyardclient_context import \ + ShipyardClientContext + +LOG = logging.getLogger(__name__) + + +class AuthValuesError(PeglegBaseException): + """Shipyard authentication failed. """ + + def __init__(self, *, diagnostic): + self.diagnostic = diagnostic + + +class DocumentUploadError(PeglegBaseException): + """ Exception occurs while uploading documents""" + + def __init__(self, message): + self.message = message + + +class ShipyardHelper(object): + """ + A helper class for Shipyard. It performs the following operation: + 1. Validates the authentication parameters required for Keystone + 2. Uploads the document to Shipyard buffer + 3. Commits the document + 4. Formats response from Shipyard api_client + """ + + def __init__(self, context): + """ + Initializes params to be used by Shipyard + + :param context: ShipyardHelper context object that contains + params for initializing ShipyardClient with + correct client context and the site_name. + """ + self.ctx = context + self.api_parameters = self.ctx.obj['API_PARAMETERS'] + self.auth_vars = self.api_parameters.get('auth_vars') + self.context_marker = self.ctx.obj['context_marker'] + if self.context_marker is None: + self.context_marker = str(uuid.uuid4()) + LOG.debug("context_marker is %s" % self.context_marker) + self.site_name = self.ctx.obj['site_name'] + self.client_context = ShipyardClientContext( + self.auth_vars, self.context_marker) + self.api_client = ShipyardClient(self.client_context) + + def upload_documents(self): + """ Uploads documents to Shipyard """ + + collected_documents = files.collect_files_by_repo(self.site_name) + + LOG.info("Uploading %s collection(s) " % len(collected_documents)) + for idx, document in enumerate(collected_documents): + # Append flag is not required for the first + # collection being uploaded to Shipyard. It + # is needed for subsequent collections. + if idx == 0: + buffer_mode = None + else: + buffer_mode = 'append' + + data = yaml.safe_dump_all(collected_documents[document]) + + try: + self.validate_auth_vars() + # Get current buffer status. + response = self.api_client.get_configdocs_status() + buff_stat = response.json() + # If buffer is empty then proceed with existing buffer value + # else pass the 'replace' flag. + for stat in range(len(buff_stat)): + if (buff_stat[stat]['new_status'] != 'unmodified' and + buffer_mode != 'append'): + buffer_mode = 'replace' + resp_text = self.api_client.post_configdocs( + collection_id=document, + buffer_mode=buffer_mode, + document_data=data + ) + + except AuthValuesError as ave: + resp_text = "Error: {}".format(ave.diagnostic) + raise DocumentUploadError(resp_text) + except Exception as ex: + resp_text = ( + "Error: Unable to invoke action due to: {}" + .format(str(ex))) + LOG.debug(resp_text, exc_info=True) + raise DocumentUploadError(resp_text) + + # FIXME: Standardize status_code in Deckhand to avoid this + # workaround. + code = 0 + if hasattr(resp_text, 'status_code'): + code = resp_text.status_code + elif hasattr(resp_text, 'code'): + code = resp_text.code + if code >= 400: + if hasattr(resp_text, 'content'): + raise DocumentUploadError(resp_text.content) + else: + raise DocumentUploadError(resp_text) + else: + output = self.formatted_response_handler(resp_text) + LOG.info("Uploaded document in buffer %s " % output) + + # Commit in the last iteration of the loop when all the documents + # have been pushed to Shipyard buffer. + if idx == len(collected_documents) - 1: + return self.commit_documents() + + def commit_documents(self): + """ Commit Shipyard buffer documents """ + + LOG.info("Commiting Shipyard buffer documents") + + try: + resp_text = self.formatted_response_handler( + self.api_client.commit_configdocs() + ) + except Exception as ex: + resp_text = ( + "Error: Unable to invoke action due to: {}".format(str(ex))) + raise DocumentUploadError(resp_text) + return resp_text + + def validate_auth_vars(self): + """Checks that the required authorization varible have been entered""" + required_auth_vars = ['auth_url'] + err_txt = [] + for var in required_auth_vars: + if self.auth_vars[var] is None: + err_txt.append( + 'Missing the required authorization variable: ' + '--os-{}'.format(var.replace('_', '-'))) + if err_txt: + for var in self.auth_vars: + if (self.auth_vars.get(var) is None and + var not in required_auth_vars): + err_txt.append('- Also not set: --os-{}'.format( + var.replace('_', '-'))) + raise AuthValuesError(diagnostic='\n'.join(err_txt)) + + def formatted_response_handler(self, response): + """Base format handler for either json or yaml depending on call""" + call = response.headers['Content-Type'] + if 'json' in call: + try: + return json.dumps(response.json(), indent=4) + except ValueError: + return ( + "This is not json and could not be printed as such. \n" + + response.text + ) diff --git a/requirements.txt b/requirements.txt index ca8d6598..b0191001 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ jsonschema==2.6.0 pyyaml==3.12 cryptography==2.3.1 git+https://github.com/openstack/airship-deckhand.git@7d697012fcbd868b14670aa9cf895acfad5a7f8d +git+https://github.com/openstack/airship-shipyard.git@44f7022df6438de541501c2fdd5c46df198b82bf#egg=shipyard_client&subdirectory=src/bin/shipyard_client diff --git a/tests/unit/engine/test_util_files.py b/tests/unit/engine/test_util_files.py index bf18a8f3..a475866d 100644 --- a/tests/unit/engine/test_util_files.py +++ b/tests/unit/engine/test_util_files.py @@ -12,10 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import mock + from pegleg import config from pegleg.engine.util import files from tests.unit.fixtures import create_tmp_deployment_files +TEST_DATA = [('/tmp/test_repo', 'test_file.yaml')] +TEST_DATA_2 = [{'schema': 'pegleg/SiteDefinition/v1', 'data': 'test'}] def test_no_non_yamls(tmpdir): p = tmpdir.mkdir("deployment_files").mkdir("global") @@ -51,3 +55,13 @@ def test_list_all_files(create_tmp_deployment_files): assert len(actual_files) == len(expected_files) for idx, file in enumerate(actual_files): assert file.endswith(expected_files[idx]) + +@mock.patch('pegleg.engine.util.definition.site_files_by_repo',autospec=True, + return_value=TEST_DATA) +@mock.patch('pegleg.engine.util.files.read', autospec=True, + return_value=TEST_DATA_2) +def test_collect_files_by_repo(*args): + result = files.collect_files_by_repo('test-site') + + assert 'test_repo' in result + assert 'schema' in result['test_repo'][0] diff --git a/tests/unit/engine/util/test_shipyard_helper.py b/tests/unit/engine/util/test_shipyard_helper.py new file mode 100644 index 00000000..ac5c3166 --- /dev/null +++ b/tests/unit/engine/util/test_shipyard_helper.py @@ -0,0 +1,172 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import mock +import pytest + +from tests.unit import test_utils +from mock import ANY + +from pegleg.engine import util +from pegleg.engine.util.shipyard_helper import ShipyardHelper +from pegleg.engine.util.shipyard_helper import ShipyardClient + +# Dummy data to be used as collected documents +DATA = {'test-repo': + {'test-data': 'RandomData'}} + +class context(): + obj = {} + + +class FakeResponse(): + code = 404 + +def _get_context(): + ctx = context() + ctx.obj = {} + auth_vars = { + 'project_domain_name': 'projDomainTest', + 'user_domain_name': 'userDomainTest', + 'project_name': 'projectTest', + 'username': 'usernameTest', + 'password': 'passwordTest', + 'auth_url': 'urlTest' + } + ctx.obj['API_PARAMETERS'] = { + 'auth_vars': auth_vars + } + ctx.obj['context_marker'] = '88888888-4444-4444-4444-121212121212' + ctx.obj['site_name'] = 'test-site' + return ctx + +def _get_bad_context(): + ctx = context() + ctx.obj = {} + auth_vars = { + 'project_domain_name': 'projDomainTest', + 'user_domain_name': 'userDomainTest', + 'project_name': 'projectTest', + 'username': 'usernameTest', + 'password': 'passwordTest', + 'auth_url': None + } + ctx.obj['API_PARAMETERS'] = { + 'auth_vars': auth_vars + } + ctx.obj['context_marker'] = '88888888-4444-4444-4444-121212121212' + ctx.obj['site_name'] = 'test-site' + return ctx + + +def test_shipyard_helper_init_(): + """ Tests ShipyardHelper init method """ + # Scenario: + # + # 1) Get a dummy context Object + # 2) Check that site name is as expected + # 3) Check api client is instance of ShipyardClient + + context = _get_context() + shipyard_helper = ShipyardHelper(context) + + assert shipyard_helper.site_name == context.obj['site_name'] + assert isinstance(shipyard_helper.api_client, ShipyardClient) + +@mock.patch('pegleg.engine.util.files.collect_files_by_repo', autospec=True, + return_value=DATA) +@mock.patch.object(ShipyardHelper, 'formatted_response_handler', + autospec=True, return_value=None) +def test_upload_documents(*args): + """ Tests upload document """ + # Scenario: + # + # 1) Get a dummy context Object + # 2) Mock external calls + # 3) Check documents uploaded to Shipyard with correct parameters + + context = _get_context() + shipyard_helper = ShipyardHelper(context) + + with mock.patch('pegleg.engine.util.shipyard_helper.ShipyardClient', + autospec=True) as mock_shipyard: + mock_api_client = mock_shipyard.return_value + mock_api_client.post_configdocs.return_value = 'Success' + result = ShipyardHelper(context).upload_documents() + + # Validate Shipyard call to post configdocs was invoked with correct + # collection name and buffer mode. + mock_api_client.post_configdocs.assert_called_with('test-repo', None, ANY) + mock_api_client.post_configdocs.assert_called_once() + +@mock.patch('pegleg.engine.util.files.collect_files_by_repo', autospec=True, + return_value=DATA) +@mock.patch.object(ShipyardHelper, 'formatted_response_handler', + autospec=True, return_value=None) +def test_upload_documents_fail(*args): + """ Tests Document upload error """ + # Scenario: + # + # 1) Get a bad context object with empty auth_url + # 2) Mock external calls + # 3) Check DocumentUploadError is raised + + context = _get_context() + shipyard_helper = ShipyardHelper(context) + + with mock.patch('pegleg.engine.util.shipyard_helper.ShipyardClient', + autospec=True) as mock_shipyard: + mock_api_client = mock_shipyard.return_value + mock_api_client.post_configdocs.return_value = FakeResponse() + with pytest.raises(util.shipyard_helper.DocumentUploadError): + ShipyardHelper(context).upload_documents() + +@mock.patch('pegleg.engine.util.files.collect_files_by_repo', autospec=True, + return_value=DATA) +@mock.patch.object(ShipyardHelper, 'formatted_response_handler', + autospec=True, return_value=None) +def test_fail_auth(*args): + """ Tests Auth Failure """ + # Scenario: + # + # 1) Get a bad context object with empty auth_url + # 2) Check AuthValuesError is raised + + context = _get_bad_context() + shipyard_helper = ShipyardHelper(context) + + with pytest.raises(util.shipyard_helper.AuthValuesError): + ShipyardHelper(context).validate_auth_vars() + +@mock.patch.object(ShipyardHelper, 'formatted_response_handler', + autospec=True, return_value=None) +def test_commit_documents(*args): + """Tests commit document """ + # Scenario: + # + # 1) Get a dummy context Object + # 2) Mock external calls + # 3) Check commit documents was called + + context = _get_context() + shipyard_helper = ShipyardHelper(context) + + with mock.patch('pegleg.engine.util.shipyard_helper.ShipyardClient', + autospec=True) as mock_shipyard: + mock_api_client = mock_shipyard.return_value + mock_api_client.commit_configdocs.return_value = 'Success' + result = ShipyardHelper(context).commit_documents() + + mock_api_client.commit_configdocs.assert_called_once() diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 4f730548..59323c4d 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -16,6 +16,7 @@ import os import shutil from click.testing import CliRunner +from mock import ANY import mock import pytest @@ -354,6 +355,22 @@ class TestSiteCliActions(BaseCLIActionTest): repo_path = self.treasuremap_path self._validate_render_site_action(repo_path) + def test_upload_documents_shipyard_using_local_repo_path(self): + """Validates ShipyardHelper is called with correct arguments.""" + # Scenario: + # + # 1) Mock out ShipyardHelper + # 2) Check ShipyardHelper was called with correct arguments + + repo_path = self.treasuremap_path + + with mock.patch('pegleg.cli.ShipyardHelper') as mock_obj: + result = self.runner.invoke(cli.site, + ['-r', repo_path, 'upload', self.site_name]) + + assert result.exit_code == 0 + mock_obj.assert_called_once() + class TestRepoCliActions(BaseCLIActionTest): """Tests repo-level CLI actions."""