CLI: Add support for uploading documents to Shipyard

This PS enables Pegleg to upload documents directly to Shipyard
thus ensuring that unencrypted data never gets stored in disk.

The flow for this new CLI command is as follows:

- Collect documents as per the provided site repository
- Decrypt the collected documets(TODO)
- Upload document to Shipyard:
  - one collection per repository will be uploaded to Shipyard

    Eg-
        pegleg site -r /opt/aic-clcp-site-manifests \
        -e global=/opt/aic-clcp-manifests upload <site-name>

        Two collections will be created in shipyard since there are two
        repositories provided. The name of the collections will be the
        name of repositories provided.
  - Commit the documents in shipyard buffer.

Change-Id: I6275252b044ebb82d8bb2009c0bea6ebf7033bce
This commit is contained in:
Nishant Kumar 2018-10-09 22:08:20 +00:00
parent e3159d223a
commit 9113d249ff
9 changed files with 498 additions and 0 deletions

View File

@ -389,6 +389,44 @@ A more complex example involves excluding certain linting checks:
lint <site_name> \
-x P001 -x P002 -w P003
Upload
-------
Uploads documents to `Shipyard`_.
**site_name** (Required).
Name of the site. The ``site_name`` must match a ``site`` name in the site
repository folder structure
**--os-<various>=<value>** (Required).
Shipyard needs these options for authenticating with OpenStack Keystone.
This option can be set as environment variables or it can be passed via
the command line.
Please reference Shipyard's `CLI documentation`_ for information related to these options.
**--context-marker=<uuid>** (Optional).
Specifies a UUID (8-4-4-4-12 format) that will be used to correlate logs,
transactions, etc. in downstream activities triggered by this interaction.
Usage:
::
./pegleg.sh site <options> upload <site_name> --context-marker=<uuid>
Examples
^^^^^^^^
::
./pegleg.sh site -r <site_repo> -e <extra_repo> \
upload <site_name> <options>
.. _command-line-repository-overrides:
Secrets
@ -641,3 +679,5 @@ P003 - All repos contain expected directories.
.. _Deckhand: https://airship-deckhand.readthedocs.io/en/latest/users/rendering.html
.. _Deckhand Validations: https://airship-deckhand.readthedocs.io/en/latest/overview.html#validation
.. _Pegleg Managed Documents: https://airship-specs.readthedocs.io/en/latest/specs/approved/pegleg-secrets.html#peglegmanageddocument
.. _Shipyard: https://github.com/openstack/airship-shipyard
.. _CLI documentation: https://airship-shipyard.readthedocs.io/en/latest/CLI.html#openstack-keystone-authorization-environment-variables

View File

@ -56,3 +56,10 @@ Git Exceptions
:members:
:show-inheritance:
:undoc-members:
Authentication Exceptions
-------------------------
.. autoexception:: pegleg.engine.util.shipyard_helper.AuthValuesError
:members:
:undoc-members:

View File

@ -20,6 +20,7 @@ import click
from pegleg import config
from pegleg import engine
from pegleg.engine.util.shipyard_helper import ShipyardHelper
LOG = logging.getLogger(__name__)
@ -325,6 +326,56 @@ def lint_site(*, fail_on_missing_sub_src, exclude_lint, warn_lint, site_name):
warn_lint=warn_lint)
@site.command('upload', help='Upload documents to Shipyard')
# Keystone authentication parameters
@click.option('--os-project-domain-name',
envvar='OS_PROJECT_DOMAIN_NAME',
required=False,
default='default')
@click.option('--os-user-domain-name',
envvar='OS_USER_DOMAIN_NAME',
required=False,
default='default')
@click.option('--os-project-name', envvar='OS_PROJECT_NAME', required=False)
@click.option('--os-username', envvar='OS_USERNAME', required=False)
@click.option('--os-password', envvar='OS_PASSWORD', required=False)
@click.option(
'--os-auth-url', envvar='OS_AUTH_URL', required=False)
# Option passed to Shipyard client context
@click.option(
'--context-marker',
help='Specifies a UUID (8-4-4-4-12 format) that will be used to correlate '
'logs, transactions, etc. in downstream activities triggered by this '
'interaction ',
required=False,
type=click.UUID)
@SITE_REPOSITORY_ARGUMENT
@click.pass_context
def upload(ctx, *, os_project_domain_name,
os_user_domain_name, os_project_name, os_username,
os_password, os_auth_url, context_marker, site_name):
if not ctx.obj:
ctx.obj = {}
# Build API parameters required by Shipyard API Client.
auth_vars = {
'project_domain_name': os_project_domain_name,
'user_domain_name': os_user_domain_name,
'project_name': os_project_name,
'username': os_username,
'password': os_password,
'auth_url': os_auth_url
}
ctx.obj['API_PARAMETERS'] = {
'auth_vars': auth_vars
}
ctx.obj['context_marker'] = str(context_marker)
ctx.obj['site_name'] = site_name
click.echo(ShipyardHelper(ctx).upload_documents())
@main.group(help='Commands related to types')
@MAIN_REPOSITORY_OPTION
@REPOSITORY_CLONE_PATH_OPTION

View File

@ -13,11 +13,13 @@
# limitations under the License.
import click
import collections
import os
import yaml
import logging
from pegleg import config
from pegleg.engine import util
from pegleg.engine.util import pegleg_managed_document as md
LOG = logging.getLogger(__name__)
@ -36,6 +38,7 @@ __all__ = [
'search',
'slurp',
'check_file_save_location',
'collect_files_by_repo',
]
DIR_DEPTHS = {
@ -366,3 +369,15 @@ def check_file_save_location(save_location):
raise click.ClickException(
'save_location %s already exists, '
'but is not a directory'.format(save_location))
def collect_files_by_repo(site_name):
""" Collects file by repo name in memory."""
collected_files_by_repo = collections.defaultdict(list)
for repo_base, filename in util.definition.site_files_by_repo(
site_name):
repo_name = os.path.normpath(repo_base).split(os.sep)[-1]
documents = util.files.read(filename)
collected_files_by_repo[repo_name].extend(documents)
return collected_files_by_repo

View File

@ -0,0 +1,181 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import logging
import uuid
import yaml
from pegleg.engine.util import files
from pegleg.engine.exceptions import PeglegBaseException
from shipyard_client.api_client.shipyard_api_client import ShipyardClient
from shipyard_client.api_client.shipyardclient_context import \
ShipyardClientContext
LOG = logging.getLogger(__name__)
class AuthValuesError(PeglegBaseException):
"""Shipyard authentication failed. """
def __init__(self, *, diagnostic):
self.diagnostic = diagnostic
class DocumentUploadError(PeglegBaseException):
""" Exception occurs while uploading documents"""
def __init__(self, message):
self.message = message
class ShipyardHelper(object):
"""
A helper class for Shipyard. It performs the following operation:
1. Validates the authentication parameters required for Keystone
2. Uploads the document to Shipyard buffer
3. Commits the document
4. Formats response from Shipyard api_client
"""
def __init__(self, context):
"""
Initializes params to be used by Shipyard
:param context: ShipyardHelper context object that contains
params for initializing ShipyardClient with
correct client context and the site_name.
"""
self.ctx = context
self.api_parameters = self.ctx.obj['API_PARAMETERS']
self.auth_vars = self.api_parameters.get('auth_vars')
self.context_marker = self.ctx.obj['context_marker']
if self.context_marker is None:
self.context_marker = str(uuid.uuid4())
LOG.debug("context_marker is %s" % self.context_marker)
self.site_name = self.ctx.obj['site_name']
self.client_context = ShipyardClientContext(
self.auth_vars, self.context_marker)
self.api_client = ShipyardClient(self.client_context)
def upload_documents(self):
""" Uploads documents to Shipyard """
collected_documents = files.collect_files_by_repo(self.site_name)
LOG.info("Uploading %s collection(s) " % len(collected_documents))
for idx, document in enumerate(collected_documents):
# Append flag is not required for the first
# collection being uploaded to Shipyard. It
# is needed for subsequent collections.
if idx == 0:
buffer_mode = None
else:
buffer_mode = 'append'
data = yaml.safe_dump_all(collected_documents[document])
try:
self.validate_auth_vars()
# Get current buffer status.
response = self.api_client.get_configdocs_status()
buff_stat = response.json()
# If buffer is empty then proceed with existing buffer value
# else pass the 'replace' flag.
for stat in range(len(buff_stat)):
if (buff_stat[stat]['new_status'] != 'unmodified' and
buffer_mode != 'append'):
buffer_mode = 'replace'
resp_text = self.api_client.post_configdocs(
collection_id=document,
buffer_mode=buffer_mode,
document_data=data
)
except AuthValuesError as ave:
resp_text = "Error: {}".format(ave.diagnostic)
raise DocumentUploadError(resp_text)
except Exception as ex:
resp_text = (
"Error: Unable to invoke action due to: {}"
.format(str(ex)))
LOG.debug(resp_text, exc_info=True)
raise DocumentUploadError(resp_text)
# FIXME: Standardize status_code in Deckhand to avoid this
# workaround.
code = 0
if hasattr(resp_text, 'status_code'):
code = resp_text.status_code
elif hasattr(resp_text, 'code'):
code = resp_text.code
if code >= 400:
if hasattr(resp_text, 'content'):
raise DocumentUploadError(resp_text.content)
else:
raise DocumentUploadError(resp_text)
else:
output = self.formatted_response_handler(resp_text)
LOG.info("Uploaded document in buffer %s " % output)
# Commit in the last iteration of the loop when all the documents
# have been pushed to Shipyard buffer.
if idx == len(collected_documents) - 1:
return self.commit_documents()
def commit_documents(self):
""" Commit Shipyard buffer documents """
LOG.info("Commiting Shipyard buffer documents")
try:
resp_text = self.formatted_response_handler(
self.api_client.commit_configdocs()
)
except Exception as ex:
resp_text = (
"Error: Unable to invoke action due to: {}".format(str(ex)))
raise DocumentUploadError(resp_text)
return resp_text
def validate_auth_vars(self):
"""Checks that the required authorization varible have been entered"""
required_auth_vars = ['auth_url']
err_txt = []
for var in required_auth_vars:
if self.auth_vars[var] is None:
err_txt.append(
'Missing the required authorization variable: '
'--os-{}'.format(var.replace('_', '-')))
if err_txt:
for var in self.auth_vars:
if (self.auth_vars.get(var) is None and
var not in required_auth_vars):
err_txt.append('- Also not set: --os-{}'.format(
var.replace('_', '-')))
raise AuthValuesError(diagnostic='\n'.join(err_txt))
def formatted_response_handler(self, response):
"""Base format handler for either json or yaml depending on call"""
call = response.headers['Content-Type']
if 'json' in call:
try:
return json.dumps(response.json(), indent=4)
except ValueError:
return (
"This is not json and could not be printed as such. \n" +
response.text
)

View File

@ -4,3 +4,4 @@ jsonschema==2.6.0
pyyaml==3.12
cryptography==2.3.1
git+https://github.com/openstack/airship-deckhand.git@7d697012fcbd868b14670aa9cf895acfad5a7f8d
git+https://github.com/openstack/airship-shipyard.git@44f7022df6438de541501c2fdd5c46df198b82bf#egg=shipyard_client&subdirectory=src/bin/shipyard_client

View File

@ -12,10 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from pegleg import config
from pegleg.engine.util import files
from tests.unit.fixtures import create_tmp_deployment_files
TEST_DATA = [('/tmp/test_repo', 'test_file.yaml')]
TEST_DATA_2 = [{'schema': 'pegleg/SiteDefinition/v1', 'data': 'test'}]
def test_no_non_yamls(tmpdir):
p = tmpdir.mkdir("deployment_files").mkdir("global")
@ -51,3 +55,13 @@ def test_list_all_files(create_tmp_deployment_files):
assert len(actual_files) == len(expected_files)
for idx, file in enumerate(actual_files):
assert file.endswith(expected_files[idx])
@mock.patch('pegleg.engine.util.definition.site_files_by_repo',autospec=True,
return_value=TEST_DATA)
@mock.patch('pegleg.engine.util.files.read', autospec=True,
return_value=TEST_DATA_2)
def test_collect_files_by_repo(*args):
result = files.collect_files_by_repo('test-site')
assert 'test_repo' in result
assert 'schema' in result['test_repo'][0]

View File

@ -0,0 +1,172 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import mock
import pytest
from tests.unit import test_utils
from mock import ANY
from pegleg.engine import util
from pegleg.engine.util.shipyard_helper import ShipyardHelper
from pegleg.engine.util.shipyard_helper import ShipyardClient
# Dummy data to be used as collected documents
DATA = {'test-repo':
{'test-data': 'RandomData'}}
class context():
obj = {}
class FakeResponse():
code = 404
def _get_context():
ctx = context()
ctx.obj = {}
auth_vars = {
'project_domain_name': 'projDomainTest',
'user_domain_name': 'userDomainTest',
'project_name': 'projectTest',
'username': 'usernameTest',
'password': 'passwordTest',
'auth_url': 'urlTest'
}
ctx.obj['API_PARAMETERS'] = {
'auth_vars': auth_vars
}
ctx.obj['context_marker'] = '88888888-4444-4444-4444-121212121212'
ctx.obj['site_name'] = 'test-site'
return ctx
def _get_bad_context():
ctx = context()
ctx.obj = {}
auth_vars = {
'project_domain_name': 'projDomainTest',
'user_domain_name': 'userDomainTest',
'project_name': 'projectTest',
'username': 'usernameTest',
'password': 'passwordTest',
'auth_url': None
}
ctx.obj['API_PARAMETERS'] = {
'auth_vars': auth_vars
}
ctx.obj['context_marker'] = '88888888-4444-4444-4444-121212121212'
ctx.obj['site_name'] = 'test-site'
return ctx
def test_shipyard_helper_init_():
""" Tests ShipyardHelper init method """
# Scenario:
#
# 1) Get a dummy context Object
# 2) Check that site name is as expected
# 3) Check api client is instance of ShipyardClient
context = _get_context()
shipyard_helper = ShipyardHelper(context)
assert shipyard_helper.site_name == context.obj['site_name']
assert isinstance(shipyard_helper.api_client, ShipyardClient)
@mock.patch('pegleg.engine.util.files.collect_files_by_repo', autospec=True,
return_value=DATA)
@mock.patch.object(ShipyardHelper, 'formatted_response_handler',
autospec=True, return_value=None)
def test_upload_documents(*args):
""" Tests upload document """
# Scenario:
#
# 1) Get a dummy context Object
# 2) Mock external calls
# 3) Check documents uploaded to Shipyard with correct parameters
context = _get_context()
shipyard_helper = ShipyardHelper(context)
with mock.patch('pegleg.engine.util.shipyard_helper.ShipyardClient',
autospec=True) as mock_shipyard:
mock_api_client = mock_shipyard.return_value
mock_api_client.post_configdocs.return_value = 'Success'
result = ShipyardHelper(context).upload_documents()
# Validate Shipyard call to post configdocs was invoked with correct
# collection name and buffer mode.
mock_api_client.post_configdocs.assert_called_with('test-repo', None, ANY)
mock_api_client.post_configdocs.assert_called_once()
@mock.patch('pegleg.engine.util.files.collect_files_by_repo', autospec=True,
return_value=DATA)
@mock.patch.object(ShipyardHelper, 'formatted_response_handler',
autospec=True, return_value=None)
def test_upload_documents_fail(*args):
""" Tests Document upload error """
# Scenario:
#
# 1) Get a bad context object with empty auth_url
# 2) Mock external calls
# 3) Check DocumentUploadError is raised
context = _get_context()
shipyard_helper = ShipyardHelper(context)
with mock.patch('pegleg.engine.util.shipyard_helper.ShipyardClient',
autospec=True) as mock_shipyard:
mock_api_client = mock_shipyard.return_value
mock_api_client.post_configdocs.return_value = FakeResponse()
with pytest.raises(util.shipyard_helper.DocumentUploadError):
ShipyardHelper(context).upload_documents()
@mock.patch('pegleg.engine.util.files.collect_files_by_repo', autospec=True,
return_value=DATA)
@mock.patch.object(ShipyardHelper, 'formatted_response_handler',
autospec=True, return_value=None)
def test_fail_auth(*args):
""" Tests Auth Failure """
# Scenario:
#
# 1) Get a bad context object with empty auth_url
# 2) Check AuthValuesError is raised
context = _get_bad_context()
shipyard_helper = ShipyardHelper(context)
with pytest.raises(util.shipyard_helper.AuthValuesError):
ShipyardHelper(context).validate_auth_vars()
@mock.patch.object(ShipyardHelper, 'formatted_response_handler',
autospec=True, return_value=None)
def test_commit_documents(*args):
"""Tests commit document """
# Scenario:
#
# 1) Get a dummy context Object
# 2) Mock external calls
# 3) Check commit documents was called
context = _get_context()
shipyard_helper = ShipyardHelper(context)
with mock.patch('pegleg.engine.util.shipyard_helper.ShipyardClient',
autospec=True) as mock_shipyard:
mock_api_client = mock_shipyard.return_value
mock_api_client.commit_configdocs.return_value = 'Success'
result = ShipyardHelper(context).commit_documents()
mock_api_client.commit_configdocs.assert_called_once()

View File

@ -16,6 +16,7 @@ import os
import shutil
from click.testing import CliRunner
from mock import ANY
import mock
import pytest
@ -354,6 +355,22 @@ class TestSiteCliActions(BaseCLIActionTest):
repo_path = self.treasuremap_path
self._validate_render_site_action(repo_path)
def test_upload_documents_shipyard_using_local_repo_path(self):
"""Validates ShipyardHelper is called with correct arguments."""
# Scenario:
#
# 1) Mock out ShipyardHelper
# 2) Check ShipyardHelper was called with correct arguments
repo_path = self.treasuremap_path
with mock.patch('pegleg.cli.ShipyardHelper') as mock_obj:
result = self.runner.invoke(cli.site,
['-r', repo_path, 'upload', self.site_name])
assert result.exit_code == 0
mock_obj.assert_called_once()
class TestRepoCliActions(BaseCLIActionTest):
"""Tests repo-level CLI actions."""