diff --git a/doc/source/cli/cli.rst b/doc/source/cli/cli.rst index b58bf9b7..69e1655b 100644 --- a/doc/source/cli/cli.rst +++ b/doc/source/cli/cli.rst @@ -389,6 +389,44 @@ A more complex example involves excluding certain linting checks: lint \ -x P001 -x P002 -w P003 +Upload +------- + +Uploads documents to `Shipyard`_. + +**site_name** (Required). + +Name of the site. The ``site_name`` must match a ``site`` name in the site +repository folder structure + +**--os-=** (Required). + +Shipyard needs these options for authenticating with OpenStack Keystone. +This option can be set as environment variables or it can be passed via +the command line. + +Please reference Shipyard's `CLI documentation`_ for information related to these options. + +**--context-marker=** (Optional). + +Specifies a UUID (8-4-4-4-12 format) that will be used to correlate logs, +transactions, etc. in downstream activities triggered by this interaction. + +Usage: + +:: + + ./pegleg.sh site upload --context-marker= + +Examples +^^^^^^^^ + +:: + + ./pegleg.sh site -r -e \ + upload + + .. _command-line-repository-overrides: Secrets @@ -641,3 +679,5 @@ P003 - All repos contain expected directories. .. _Deckhand: https://airship-deckhand.readthedocs.io/en/latest/users/rendering.html .. _Deckhand Validations: https://airship-deckhand.readthedocs.io/en/latest/overview.html#validation .. _Pegleg Managed Documents: https://airship-specs.readthedocs.io/en/latest/specs/approved/pegleg-secrets.html#peglegmanageddocument +.. _Shipyard: https://github.com/openstack/airship-shipyard +.. _CLI documentation: https://airship-shipyard.readthedocs.io/en/latest/CLI.html#openstack-keystone-authorization-environment-variables diff --git a/doc/source/exceptions.rst b/doc/source/exceptions.rst index a0c5450f..8fb8577a 100644 --- a/doc/source/exceptions.rst +++ b/doc/source/exceptions.rst @@ -56,3 +56,10 @@ Git Exceptions :members: :show-inheritance: :undoc-members: + +Authentication Exceptions +------------------------- + +.. autoexception:: pegleg.engine.util.shipyard_helper.AuthValuesError + :members: + :undoc-members: diff --git a/pegleg/cli.py b/pegleg/cli.py index 84b03e4f..c7b10192 100644 --- a/pegleg/cli.py +++ b/pegleg/cli.py @@ -20,6 +20,7 @@ import click from pegleg import config from pegleg import engine +from pegleg.engine.util.shipyard_helper import ShipyardHelper LOG = logging.getLogger(__name__) @@ -325,6 +326,56 @@ def lint_site(*, fail_on_missing_sub_src, exclude_lint, warn_lint, site_name): warn_lint=warn_lint) +@site.command('upload', help='Upload documents to Shipyard') +# Keystone authentication parameters +@click.option('--os-project-domain-name', + envvar='OS_PROJECT_DOMAIN_NAME', + required=False, + default='default') +@click.option('--os-user-domain-name', + envvar='OS_USER_DOMAIN_NAME', + required=False, + default='default') +@click.option('--os-project-name', envvar='OS_PROJECT_NAME', required=False) +@click.option('--os-username', envvar='OS_USERNAME', required=False) +@click.option('--os-password', envvar='OS_PASSWORD', required=False) +@click.option( + '--os-auth-url', envvar='OS_AUTH_URL', required=False) +# Option passed to Shipyard client context +@click.option( + '--context-marker', + help='Specifies a UUID (8-4-4-4-12 format) that will be used to correlate ' + 'logs, transactions, etc. in downstream activities triggered by this ' + 'interaction ', + required=False, + type=click.UUID) +@SITE_REPOSITORY_ARGUMENT +@click.pass_context +def upload(ctx, *, os_project_domain_name, + os_user_domain_name, os_project_name, os_username, + os_password, os_auth_url, context_marker, site_name): + if not ctx.obj: + ctx.obj = {} + + # Build API parameters required by Shipyard API Client. + auth_vars = { + 'project_domain_name': os_project_domain_name, + 'user_domain_name': os_user_domain_name, + 'project_name': os_project_name, + 'username': os_username, + 'password': os_password, + 'auth_url': os_auth_url + } + + ctx.obj['API_PARAMETERS'] = { + 'auth_vars': auth_vars + } + ctx.obj['context_marker'] = str(context_marker) + ctx.obj['site_name'] = site_name + + click.echo(ShipyardHelper(ctx).upload_documents()) + + @main.group(help='Commands related to types') @MAIN_REPOSITORY_OPTION @REPOSITORY_CLONE_PATH_OPTION diff --git a/pegleg/engine/util/files.py b/pegleg/engine/util/files.py index 0ab0cae9..7a182851 100644 --- a/pegleg/engine/util/files.py +++ b/pegleg/engine/util/files.py @@ -13,11 +13,13 @@ # limitations under the License. import click +import collections import os import yaml import logging from pegleg import config +from pegleg.engine import util from pegleg.engine.util import pegleg_managed_document as md LOG = logging.getLogger(__name__) @@ -36,6 +38,7 @@ __all__ = [ 'search', 'slurp', 'check_file_save_location', + 'collect_files_by_repo', ] DIR_DEPTHS = { @@ -366,3 +369,15 @@ def check_file_save_location(save_location): raise click.ClickException( 'save_location %s already exists, ' 'but is not a directory'.format(save_location)) + + +def collect_files_by_repo(site_name): + """ Collects file by repo name in memory.""" + + collected_files_by_repo = collections.defaultdict(list) + for repo_base, filename in util.definition.site_files_by_repo( + site_name): + repo_name = os.path.normpath(repo_base).split(os.sep)[-1] + documents = util.files.read(filename) + collected_files_by_repo[repo_name].extend(documents) + return collected_files_by_repo diff --git a/pegleg/engine/util/shipyard_helper.py b/pegleg/engine/util/shipyard_helper.py new file mode 100644 index 00000000..4c58580e --- /dev/null +++ b/pegleg/engine/util/shipyard_helper.py @@ -0,0 +1,181 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import uuid + +import yaml + +from pegleg.engine.util import files +from pegleg.engine.exceptions import PeglegBaseException + +from shipyard_client.api_client.shipyard_api_client import ShipyardClient +from shipyard_client.api_client.shipyardclient_context import \ + ShipyardClientContext + +LOG = logging.getLogger(__name__) + + +class AuthValuesError(PeglegBaseException): + """Shipyard authentication failed. """ + + def __init__(self, *, diagnostic): + self.diagnostic = diagnostic + + +class DocumentUploadError(PeglegBaseException): + """ Exception occurs while uploading documents""" + + def __init__(self, message): + self.message = message + + +class ShipyardHelper(object): + """ + A helper class for Shipyard. It performs the following operation: + 1. Validates the authentication parameters required for Keystone + 2. Uploads the document to Shipyard buffer + 3. Commits the document + 4. Formats response from Shipyard api_client + """ + + def __init__(self, context): + """ + Initializes params to be used by Shipyard + + :param context: ShipyardHelper context object that contains + params for initializing ShipyardClient with + correct client context and the site_name. + """ + self.ctx = context + self.api_parameters = self.ctx.obj['API_PARAMETERS'] + self.auth_vars = self.api_parameters.get('auth_vars') + self.context_marker = self.ctx.obj['context_marker'] + if self.context_marker is None: + self.context_marker = str(uuid.uuid4()) + LOG.debug("context_marker is %s" % self.context_marker) + self.site_name = self.ctx.obj['site_name'] + self.client_context = ShipyardClientContext( + self.auth_vars, self.context_marker) + self.api_client = ShipyardClient(self.client_context) + + def upload_documents(self): + """ Uploads documents to Shipyard """ + + collected_documents = files.collect_files_by_repo(self.site_name) + + LOG.info("Uploading %s collection(s) " % len(collected_documents)) + for idx, document in enumerate(collected_documents): + # Append flag is not required for the first + # collection being uploaded to Shipyard. It + # is needed for subsequent collections. + if idx == 0: + buffer_mode = None + else: + buffer_mode = 'append' + + data = yaml.safe_dump_all(collected_documents[document]) + + try: + self.validate_auth_vars() + # Get current buffer status. + response = self.api_client.get_configdocs_status() + buff_stat = response.json() + # If buffer is empty then proceed with existing buffer value + # else pass the 'replace' flag. + for stat in range(len(buff_stat)): + if (buff_stat[stat]['new_status'] != 'unmodified' and + buffer_mode != 'append'): + buffer_mode = 'replace' + resp_text = self.api_client.post_configdocs( + collection_id=document, + buffer_mode=buffer_mode, + document_data=data + ) + + except AuthValuesError as ave: + resp_text = "Error: {}".format(ave.diagnostic) + raise DocumentUploadError(resp_text) + except Exception as ex: + resp_text = ( + "Error: Unable to invoke action due to: {}" + .format(str(ex))) + LOG.debug(resp_text, exc_info=True) + raise DocumentUploadError(resp_text) + + # FIXME: Standardize status_code in Deckhand to avoid this + # workaround. + code = 0 + if hasattr(resp_text, 'status_code'): + code = resp_text.status_code + elif hasattr(resp_text, 'code'): + code = resp_text.code + if code >= 400: + if hasattr(resp_text, 'content'): + raise DocumentUploadError(resp_text.content) + else: + raise DocumentUploadError(resp_text) + else: + output = self.formatted_response_handler(resp_text) + LOG.info("Uploaded document in buffer %s " % output) + + # Commit in the last iteration of the loop when all the documents + # have been pushed to Shipyard buffer. + if idx == len(collected_documents) - 1: + return self.commit_documents() + + def commit_documents(self): + """ Commit Shipyard buffer documents """ + + LOG.info("Commiting Shipyard buffer documents") + + try: + resp_text = self.formatted_response_handler( + self.api_client.commit_configdocs() + ) + except Exception as ex: + resp_text = ( + "Error: Unable to invoke action due to: {}".format(str(ex))) + raise DocumentUploadError(resp_text) + return resp_text + + def validate_auth_vars(self): + """Checks that the required authorization varible have been entered""" + required_auth_vars = ['auth_url'] + err_txt = [] + for var in required_auth_vars: + if self.auth_vars[var] is None: + err_txt.append( + 'Missing the required authorization variable: ' + '--os-{}'.format(var.replace('_', '-'))) + if err_txt: + for var in self.auth_vars: + if (self.auth_vars.get(var) is None and + var not in required_auth_vars): + err_txt.append('- Also not set: --os-{}'.format( + var.replace('_', '-'))) + raise AuthValuesError(diagnostic='\n'.join(err_txt)) + + def formatted_response_handler(self, response): + """Base format handler for either json or yaml depending on call""" + call = response.headers['Content-Type'] + if 'json' in call: + try: + return json.dumps(response.json(), indent=4) + except ValueError: + return ( + "This is not json and could not be printed as such. \n" + + response.text + ) diff --git a/requirements.txt b/requirements.txt index ca8d6598..b0191001 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ jsonschema==2.6.0 pyyaml==3.12 cryptography==2.3.1 git+https://github.com/openstack/airship-deckhand.git@7d697012fcbd868b14670aa9cf895acfad5a7f8d +git+https://github.com/openstack/airship-shipyard.git@44f7022df6438de541501c2fdd5c46df198b82bf#egg=shipyard_client&subdirectory=src/bin/shipyard_client diff --git a/tests/unit/engine/test_util_files.py b/tests/unit/engine/test_util_files.py index bf18a8f3..a475866d 100644 --- a/tests/unit/engine/test_util_files.py +++ b/tests/unit/engine/test_util_files.py @@ -12,10 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import mock + from pegleg import config from pegleg.engine.util import files from tests.unit.fixtures import create_tmp_deployment_files +TEST_DATA = [('/tmp/test_repo', 'test_file.yaml')] +TEST_DATA_2 = [{'schema': 'pegleg/SiteDefinition/v1', 'data': 'test'}] def test_no_non_yamls(tmpdir): p = tmpdir.mkdir("deployment_files").mkdir("global") @@ -51,3 +55,13 @@ def test_list_all_files(create_tmp_deployment_files): assert len(actual_files) == len(expected_files) for idx, file in enumerate(actual_files): assert file.endswith(expected_files[idx]) + +@mock.patch('pegleg.engine.util.definition.site_files_by_repo',autospec=True, + return_value=TEST_DATA) +@mock.patch('pegleg.engine.util.files.read', autospec=True, + return_value=TEST_DATA_2) +def test_collect_files_by_repo(*args): + result = files.collect_files_by_repo('test-site') + + assert 'test_repo' in result + assert 'schema' in result['test_repo'][0] diff --git a/tests/unit/engine/util/test_shipyard_helper.py b/tests/unit/engine/util/test_shipyard_helper.py new file mode 100644 index 00000000..ac5c3166 --- /dev/null +++ b/tests/unit/engine/util/test_shipyard_helper.py @@ -0,0 +1,172 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import mock +import pytest + +from tests.unit import test_utils +from mock import ANY + +from pegleg.engine import util +from pegleg.engine.util.shipyard_helper import ShipyardHelper +from pegleg.engine.util.shipyard_helper import ShipyardClient + +# Dummy data to be used as collected documents +DATA = {'test-repo': + {'test-data': 'RandomData'}} + +class context(): + obj = {} + + +class FakeResponse(): + code = 404 + +def _get_context(): + ctx = context() + ctx.obj = {} + auth_vars = { + 'project_domain_name': 'projDomainTest', + 'user_domain_name': 'userDomainTest', + 'project_name': 'projectTest', + 'username': 'usernameTest', + 'password': 'passwordTest', + 'auth_url': 'urlTest' + } + ctx.obj['API_PARAMETERS'] = { + 'auth_vars': auth_vars + } + ctx.obj['context_marker'] = '88888888-4444-4444-4444-121212121212' + ctx.obj['site_name'] = 'test-site' + return ctx + +def _get_bad_context(): + ctx = context() + ctx.obj = {} + auth_vars = { + 'project_domain_name': 'projDomainTest', + 'user_domain_name': 'userDomainTest', + 'project_name': 'projectTest', + 'username': 'usernameTest', + 'password': 'passwordTest', + 'auth_url': None + } + ctx.obj['API_PARAMETERS'] = { + 'auth_vars': auth_vars + } + ctx.obj['context_marker'] = '88888888-4444-4444-4444-121212121212' + ctx.obj['site_name'] = 'test-site' + return ctx + + +def test_shipyard_helper_init_(): + """ Tests ShipyardHelper init method """ + # Scenario: + # + # 1) Get a dummy context Object + # 2) Check that site name is as expected + # 3) Check api client is instance of ShipyardClient + + context = _get_context() + shipyard_helper = ShipyardHelper(context) + + assert shipyard_helper.site_name == context.obj['site_name'] + assert isinstance(shipyard_helper.api_client, ShipyardClient) + +@mock.patch('pegleg.engine.util.files.collect_files_by_repo', autospec=True, + return_value=DATA) +@mock.patch.object(ShipyardHelper, 'formatted_response_handler', + autospec=True, return_value=None) +def test_upload_documents(*args): + """ Tests upload document """ + # Scenario: + # + # 1) Get a dummy context Object + # 2) Mock external calls + # 3) Check documents uploaded to Shipyard with correct parameters + + context = _get_context() + shipyard_helper = ShipyardHelper(context) + + with mock.patch('pegleg.engine.util.shipyard_helper.ShipyardClient', + autospec=True) as mock_shipyard: + mock_api_client = mock_shipyard.return_value + mock_api_client.post_configdocs.return_value = 'Success' + result = ShipyardHelper(context).upload_documents() + + # Validate Shipyard call to post configdocs was invoked with correct + # collection name and buffer mode. + mock_api_client.post_configdocs.assert_called_with('test-repo', None, ANY) + mock_api_client.post_configdocs.assert_called_once() + +@mock.patch('pegleg.engine.util.files.collect_files_by_repo', autospec=True, + return_value=DATA) +@mock.patch.object(ShipyardHelper, 'formatted_response_handler', + autospec=True, return_value=None) +def test_upload_documents_fail(*args): + """ Tests Document upload error """ + # Scenario: + # + # 1) Get a bad context object with empty auth_url + # 2) Mock external calls + # 3) Check DocumentUploadError is raised + + context = _get_context() + shipyard_helper = ShipyardHelper(context) + + with mock.patch('pegleg.engine.util.shipyard_helper.ShipyardClient', + autospec=True) as mock_shipyard: + mock_api_client = mock_shipyard.return_value + mock_api_client.post_configdocs.return_value = FakeResponse() + with pytest.raises(util.shipyard_helper.DocumentUploadError): + ShipyardHelper(context).upload_documents() + +@mock.patch('pegleg.engine.util.files.collect_files_by_repo', autospec=True, + return_value=DATA) +@mock.patch.object(ShipyardHelper, 'formatted_response_handler', + autospec=True, return_value=None) +def test_fail_auth(*args): + """ Tests Auth Failure """ + # Scenario: + # + # 1) Get a bad context object with empty auth_url + # 2) Check AuthValuesError is raised + + context = _get_bad_context() + shipyard_helper = ShipyardHelper(context) + + with pytest.raises(util.shipyard_helper.AuthValuesError): + ShipyardHelper(context).validate_auth_vars() + +@mock.patch.object(ShipyardHelper, 'formatted_response_handler', + autospec=True, return_value=None) +def test_commit_documents(*args): + """Tests commit document """ + # Scenario: + # + # 1) Get a dummy context Object + # 2) Mock external calls + # 3) Check commit documents was called + + context = _get_context() + shipyard_helper = ShipyardHelper(context) + + with mock.patch('pegleg.engine.util.shipyard_helper.ShipyardClient', + autospec=True) as mock_shipyard: + mock_api_client = mock_shipyard.return_value + mock_api_client.commit_configdocs.return_value = 'Success' + result = ShipyardHelper(context).commit_documents() + + mock_api_client.commit_configdocs.assert_called_once() diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 4f730548..59323c4d 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -16,6 +16,7 @@ import os import shutil from click.testing import CliRunner +from mock import ANY import mock import pytest @@ -354,6 +355,22 @@ class TestSiteCliActions(BaseCLIActionTest): repo_path = self.treasuremap_path self._validate_render_site_action(repo_path) + def test_upload_documents_shipyard_using_local_repo_path(self): + """Validates ShipyardHelper is called with correct arguments.""" + # Scenario: + # + # 1) Mock out ShipyardHelper + # 2) Check ShipyardHelper was called with correct arguments + + repo_path = self.treasuremap_path + + with mock.patch('pegleg.cli.ShipyardHelper') as mock_obj: + result = self.runner.invoke(cli.site, + ['-r', repo_path, 'upload', self.site_name]) + + assert result.exit_code == 0 + mock_obj.assert_called_once() + class TestRepoCliActions(BaseCLIActionTest): """Tests repo-level CLI actions."""