airship
/
shipyard
Code Issues Proposed changes

Shipyard API for Configdocs Status 

Shipyard API to discover buffered and committed collections of configdocs

1 of 4 Commits (API Client, CLI, Documentation are in separate commits)

API
-added url to api.py and policy.py
-added ConfigDocsStatusResource for on_get call in configdocs_api.py
-obtain the status of the configdocs in configdocs_helper.py
-unit tests are located in test_configdocs_api.py and test_configdocs_helper.py

Change-Id: Iaf3f6d9113ae778c5efd8b3df1a8ebd16c385c14
This commit is contained in:
One-Fine-Day 2017-12-27 13:39:48 -06:00
parent 609bc0a624
commit 9aa13ac5e9
6 changed files with 465 additions and 430 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
shipyard_airflow/control/api.py
View File

@ -29,7 +29,8 @@ from shipyard_airflow.control.af_monitoring.workflows_api import (
from shipyard_airflow.control.base import BaseResource, ShipyardRequest
from shipyard_airflow.control.configdocs.configdocs_api import (
CommitConfigDocsResource,
ConfigDocsResource
ConfigDocsResource,
ConfigDocsStatusResource
)
from shipyard_airflow.control.configdocs.rendered_configdocs_api import \
RenderedConfigDocsResource
@ -66,6 +67,7 @@ def start_api():
ActionsStepsResource()),
('/actions/{action_id}/validations/{validation_id}',
ActionsValidationsResource()),
('/configdocs', ConfigDocsStatusResource()),
('/configdocs/{collection_id}', ConfigDocsResource()),
('/commitconfigdocs', CommitConfigDocsResource()),
('/renderedconfigdocs', RenderedConfigDocsResource()),

59
shipyard_airflow/control/configdocs/configdocs_api.py
View File

@ -22,15 +22,27 @@ from shipyard_airflow.control.configdocs import configdocs_helper
from shipyard_airflow.control.api_lock import (api_lock, ApiLockType)
from shipyard_airflow.control.base import BaseResource
from shipyard_airflow.control.configdocs.configdocs_helper import (
BufferMode,
ConfigdocsHelper
)
BufferMode, ConfigdocsHelper)
from shipyard_airflow.errors import ApiError
CONF = cfg.CONF
VERSION_VALUES = ['buffer', 'committed']
class ConfigDocsStatusResource(BaseResource):
"""
Configdocs Status handles the retrieval of the configuration documents'
statuses
"""
@policy.ApiEnforcer('workflow_orchestrator:get_configdocs_status')
def on_get(self, req, resp):
"""Returns a list of the configdocs and their statuses"""
helper = ConfigdocsHelper(req.context)
resp.body = helper.get_configdocs_status()
resp.status = falcon.HTTP_200
class ConfigDocsResource(BaseResource):
"""
Configdocs handles the creation and retrieval of configuration
@ -49,8 +61,7 @@ class ConfigDocsResource(BaseResource):
helper=helper,
collection_id=collection_id,
document_data=document_data,
buffer_mode_param=req.params.get('buffermode')
)
buffer_mode_param=req.params.get('buffermode'))
resp.location = '/api/v1.0/configdocs/{}'.format(collection_id)
resp.body = self.to_json(validations)
resp.status = falcon.HTTP_201
@ -65,10 +76,7 @@ class ConfigDocsResource(BaseResource):
helper = ConfigdocsHelper(req.context)
# Not reformatting to JSON or YAML since just passing through
resp.body = self.get_collection(
helper=helper,
collection_id=collection_id,
version=version
)
helper=helper, collection_id=collection_id, version=version)
resp.append_header('Content-Type', 'application/x-yaml')
resp.status = falcon.HTTP_200
@ -78,16 +86,11 @@ class ConfigDocsResource(BaseResource):
raise ApiError(
title='Invalid version query parameter specified',
description=(
'version must be {}'.format(', '.join(VERSION_VALUES))
),
'version must be {}'.format(', '.join(VERSION_VALUES))),
status=falcon.HTTP_400,
retry=False,
)
retry=False, )
def get_collection(self,
helper,
collection_id,
version='buffer'):
def get_collection(self, helper, collection_id, version='buffer'):
"""
Attempts to retrieve the specified collection of documents
either from the buffer or committed version, as specified
@ -107,15 +110,10 @@ class ConfigDocsResource(BaseResource):
else:
buffer_mode = ConfigdocsHelper.get_buffer_mode(buffer_mode_param)
if helper.is_buffer_valid_for_bucket(collection_id,
buffer_mode):
buffer_revision = helper.add_collection(
collection_id,
document_data
)
return helper.get_deckhand_validation_status(
buffer_revision
)
if helper.is_buffer_valid_for_bucket(collection_id, buffer_mode):
buffer_revision = helper.add_collection(collection_id,
document_data)
return helper.get_deckhand_validation_status(buffer_revision)
else:
raise ApiError(
title='Invalid collection specified for buffer',
@ -127,8 +125,7 @@ class ConfigDocsResource(BaseResource):
'Setting a different buffermode may '
'provide the desired functionality')
}],
retry=False,
)
retry=False, )
class CommitConfigDocsResource(BaseResource):
@ -163,8 +160,7 @@ class CommitConfigDocsResource(BaseResource):
title=CommitConfigDocsResource.unable_to_commmit,
description='There are no documents in the buffer to commit',
status=falcon.HTTP_409,
retry=True
)
retry=True)
validations = helper.get_validations_for_buffer()
if force or validations.get('status') == 'Success':
helper.tag_buffer(configdocs_helper.COMMITTED)
@ -173,8 +169,7 @@ class CommitConfigDocsResource(BaseResource):
validations['code'] = falcon.HTTP_200
if validations.get('message'):
validations['message'] = (
validations['message'] + ' FORCED SUCCESS'
)
validations['message'] + ' FORCED SUCCESS')
else:
validations['message'] = 'FORCED SUCCESS'
return validations

272
shipyard_airflow/control/configdocs/configdocs_helper.py
View File

@ -26,18 +26,10 @@ from oslo_config import cfg
import requests
from shipyard_airflow.control.configdocs.deckhand_client import (
DeckhandClient,
DeckhandPaths,
DeckhandRejectedInputError,
DeckhandResponseError,
DocumentExistsElsewhereError,
NoRevisionsExistError
)
DeckhandClient, DeckhandPaths, DeckhandRejectedInputError,
DeckhandResponseError, DocumentExistsElsewhereError, NoRevisionsExistError)
from shipyard_airflow.control.service_endpoints import (
Endpoints,
get_endpoint,
get_token
)
Endpoints, get_endpoint, get_token)
from shipyard_airflow.errors import ApiError, AppError
CONF = cfg.CONF
@ -112,28 +104,23 @@ class ConfigdocsHelper(object):
# If there is no committed revision, then it's 0.
# new revision is ok because we just checked for buffer emptiness
old_revision_id = self._get_committed_rev_id()
if old_revision_id is None:
old_revision_id = 0
old_revision_id = self._get_committed_rev_id() or 0
try:
diff = self.deckhand.get_diff(
old_revision_id=old_revision_id,
new_revision_id=self._get_buffer_rev_id()
)
new_revision_id=self._get_buffer_rev_id())
# the collection is in the buffer if it's not unmodified
return diff.get(collection_id, 'unmodified') != 'unmodified'
except DeckhandResponseError as drex:
raise AppError(
title='Unable to retrieve revisions',
description=(
'Deckhand has responded unexpectedly: {}:{}'.format(
drex.status_code,
drex.response_message
)
),
drex.status_code, drex.response_message)),
status=falcon.HTTP_500,
retry=False,
)
retry=False, )
def is_buffer_valid_for_bucket(self, collection_id, buffermode):
"""
@ -166,6 +153,56 @@ class ConfigdocsHelper(object):
self.deckhand.rollback(committed_rev_id)
return True
def get_configdocs_status(self):
"""
Returns a list of the configdocs, committed or in buffer, and their
current committed and buffer statuses
"""
configdocs_status = []
# If there is no committed revision, then it's 0.
# new revision is ok because we just checked for buffer emptiness
old_revision_id = self._get_committed_rev_id() or 0
new_revision_id = self._get_buffer_rev_id() or old_revision_id
try:
diff = self.deckhand.get_diff(
old_revision_id=old_revision_id,
new_revision_id=new_revision_id)
except DeckhandResponseError as drex:
raise AppError(
title='Unable to retrieve revisions',
description=(
'Deckhand has responded unexpectedly: {}:{}'.format(
drex.status_code, drex.response_message)),
status=falcon.HTTP_500,
retry=False, )
for collection_id in diff:
collection = {"collection_name": collection_id}
if diff[collection_id] in [
"unmodified", "modified", "created", "deleted"]:
collection['buffer_status'] = diff[collection_id]
if diff[collection_id] == "created":
collection['committed_status'] = 'not present'
else:
collection['committed_status'] = 'present'
else:
raise AppError(
title='Invalid collection status',
description=(
'Collection_id, {} has an invalid collection status. '
'unmodified, modified, created, and deleted are the'
' only valid collection statuses.',
collection_id),
status=falcon.HTTP_500,
retry=False, )
configdocs_status.append(collection)
return configdocs_status
def _get_revision_dict(self):
"""
Returns a dictionary with values representing the revisions in
@ -209,13 +246,9 @@ class ConfigdocsHelper(object):
title='Unable to retrieve revisions',
description=(
'Deckhand has responded unexpectedly: {}:{}'.format(
drex.status_code,
drex.response_message
)
),
drex.status_code, drex.response_message)),
status=falcon.HTTP_500,
retry=False,
)
retry=False)
self.revision_dict = {
COMMITTED: committed_revision,
BUFFER: buffer_revision,
@ -277,16 +310,13 @@ class ConfigdocsHelper(object):
# revision exists
buffer_id = self._get_buffer_rev_id()
return self.deckhand.get_docs_from_revision(
revision_id=buffer_id,
bucket_id=collection_id
)
revision_id=buffer_id, bucket_id=collection_id)
raise ApiError(
title='No documents to retrieve',
description=('The Shipyard buffer is empty or does not contain '
'this collection'),
status=falcon.HTTP_404,
retry=False,
)
retry=False)
def _get_committed_docs(self, collection_id):
"""
@ -295,16 +325,13 @@ class ConfigdocsHelper(object):
committed_id = self._get_committed_rev_id()
if committed_id:
return self.deckhand.get_docs_from_revision(
revision_id=committed_id,
bucket_id=collection_id
)
revision_id=committed_id, bucket_id=collection_id)
# if there is no committed...
raise ApiError(
title='No documents to retrieve',
description='There is no committed version of this collection',
status=falcon.HTTP_404,
retry=False,
)
retry=False)
def get_rendered_configdocs(self, version=BUFFER):
"""
@ -316,15 +343,13 @@ class ConfigdocsHelper(object):
if revision_dict.get(version):
revision_id = revision_dict.get(version).get('id')
return self.deckhand.get_rendered_docs_from_revision(
revision_id=revision_id
)
revision_id=revision_id)
else:
raise ApiError(
title='This revision does not exist',
description='{} version does not exist'.format(version),
status=falcon.HTTP_404,
retry=False,
)
retry=False)
def get_validations_for_buffer(self):
"""
@ -338,17 +363,16 @@ class ConfigdocsHelper(object):
description=('Buffer revision id could not be determined from'
'Deckhand'),
status=falcon.HTTP_500,
retry=False,
)
retry=False)
@staticmethod
def _get_design_reference(revision_id):
# Constructs the design reference as json for use by other components
design_reference = {
"rel": "design",
"href": "deckhand+{}".format(DeckhandClient.get_path(
DeckhandPaths.RENDERED_REVISION_DOCS).format(revision_id)
),
"href": "deckhand+{}".format(
DeckhandClient.get_path(DeckhandPaths.RENDERED_REVISION_DOCS)
.format(revision_id)),
"type": "application/x-yaml"
}
return json.dumps(design_reference)
@ -358,16 +382,18 @@ class ConfigdocsHelper(object):
# returns the list of validation endpoint supported
val_ep = '{}/validatedesign'
return [
{'name': 'Drydock',
'url': val_ep.format(get_endpoint(Endpoints.DRYDOCK))},
{'name': 'Armada',
'url': val_ep.format(get_endpoint(Endpoints.ARMADA))},
{
'name': 'Drydock',
'url': val_ep.format(get_endpoint(Endpoints.DRYDOCK))
},
{
'name': 'Armada',
'url': val_ep.format(get_endpoint(Endpoints.ARMADA))
},
]
@staticmethod
def _get_validation_threads(validation_endpoints,
revision_id,
ctx):
def _get_validation_threads(validation_endpoints, revision_id, ctx):
# create a list of validation threads from the endpoints
validation_threads = []
for endpoint in validation_endpoints:
@ -375,39 +401,33 @@ class ConfigdocsHelper(object):
response = {'response': None}
exception = {'exception': None}
design_ref = ConfigdocsHelper._get_design_reference(revision_id)
validation_threads.append(
{
'thread': threading.Thread(
target=ConfigdocsHelper._get_validations_for_component,
kwargs={
'url': endpoint['url'],
'design_reference': design_ref,
'response': response,
'exception': exception,
'context_marker': ctx.external_marker,
'thread_name': endpoint['name'],
'log_extra': {
'req_id': ctx.request_id,
'external_ctx': ctx.external_marker,
'user': ctx.user
}
validation_threads.append({
'thread':
threading.Thread(
target=ConfigdocsHelper._get_validations_for_component,
kwargs={
'url': endpoint['url'],
'design_reference': design_ref,
'response': response,
'exception': exception,
'context_marker': ctx.external_marker,
'thread_name': endpoint['name'],
'log_extra': {
'req_id': ctx.request_id,
'external_ctx': ctx.external_marker,
'user': ctx.user
}
),
'name': endpoint['name'],
'url': endpoint['url'],
'response': response,
'exception': exception
}
)
}),
'name': endpoint['name'],
'url': endpoint['url'],
'response': response,
'exception': exception
})
return validation_threads
@staticmethod
def _get_validations_for_component(url,
design_reference,
response,
exception,
context_marker,
thread_name,
def _get_validations_for_component(url, design_reference, response,
exception, context_marker, thread_name,
**kwargs):
# Invoke the POST for validation
try:
@ -417,10 +437,8 @@ class ConfigdocsHelper(object):
'content-type': 'application/json'
}
http_resp = requests.post(url,
headers=headers,
data=design_reference,
timeout=(5, 30))
http_resp = requests.post(
url, headers=headers, data=design_reference, timeout=(5, 30))
# 400 response is "valid" failure to validate. > 400 is a problem.
if http_resp.status_code > 400:
http_resp.raise_for_status()
@ -433,16 +451,13 @@ class ConfigdocsHelper(object):
LOG.error(str(ex))
response['response'] = {
'details': {
'messageList': [
{
'message': unable_str,
'error': True
},
{
'message': str(ex),
'error': True
}
]
'messageList': [{
'message': unable_str,
'error': True
}, {
'message': str(ex),
'error': True
}]
}
}
exception['exception'] = ex
@ -457,10 +472,8 @@ class ConfigdocsHelper(object):
resp_msgs = []
validation_threads = ConfigdocsHelper._get_validation_threads(
ConfigdocsHelper._get_validation_endpoints(),
revision_id,
self.ctx
)
ConfigdocsHelper._get_validation_endpoints(), revision_id,
self.ctx)
# trigger each validation in parallel
for validation_thread in validation_threads:
if validation_thread.get('thread'):
@ -475,8 +488,8 @@ class ConfigdocsHelper(object):
th_name = validation_thread.get('name')
val_response = validation_thread.get('response',
{}).get('response')
LOG.debug("Validation from: %s response: %s",
th_name, str(val_response))
LOG.debug("Validation from: %s response: %s", th_name,
str(val_response))
if validation_thread.get('exception', {}).get('exception'):
LOG.error('Invocation of validation by %s has failed', th_name)
# invalid status needs collection of messages
@ -488,21 +501,17 @@ class ConfigdocsHelper(object):
for msg in msg_list:
if msg.get('error'):
error_count = error_count + 1
resp_msgs.append(
{
'name': th_name,
'message': msg.get('message'),
'error': True
}
)
resp_msgs.append({
'name': th_name,
'message': msg.get('message'),
'error': True
})
else:
resp_msgs.append(
{
'name': th_name,
'message': msg.get('message'),
'error': False
}
)
resp_msgs.append({
'name': th_name,
'message': msg.get('message'),
'error': False
})
# Deckhand does it differently. Incorporate those validation
# failures
dh_validations = self._get_deckhand_validations(revision_id)
@ -510,9 +519,7 @@ class ConfigdocsHelper(object):
resp_msgs.extend(dh_validations)
# return the formatted status response
return ConfigdocsHelper._format_validations_to_status(
resp_msgs,
error_count
)
resp_msgs, error_count)
def get_deckhand_validation_status(self, revision_id):
"""
@ -522,9 +529,7 @@ class ConfigdocsHelper(object):
dh_validations = self._get_deckhand_validations(revision_id)
error_count = len(dh_validations)
return ConfigdocsHelper._format_validations_to_status(
dh_validations,
error_count
)
dh_validations, error_count)
def _get_deckhand_validations(self, revision_id):
# Returns any validations that deckhand has on hand for this
@ -535,13 +540,11 @@ class ConfigdocsHelper(object):
for dh_result in deckhand_val.get('results'):
if dh_result.get('errors'):
for error in dh_result.get('errors'):
resp_msgs.append(
{
'name': dh_result.get('name'),
'message': error.get('message'),
'error': True
}
)
resp_msgs.append({
'name': dh_result.get('name'),
'message': error.get('message'),
'error': True
})
return resp_msgs
@staticmethod
@ -582,8 +585,7 @@ class ConfigdocsHelper(object):
description=('Buffer revision id could not be determined from'
'Deckhand'),
status=falcon.HTTP_500,
retry=False,
)
retry=False)
self.tag_revision(buffer_rev_id, tag)
def tag_revision(self, revision_id, tag):
@ -607,15 +609,13 @@ class ConfigdocsHelper(object):
raise ApiError(
title='Documents may not exist in more than one collection',
description=deee.response_message,
status=falcon.HTTP_409
)
status=falcon.HTTP_409)
except DeckhandRejectedInputError as drie:
LOG.info('Deckhand has rejected this input because: %s',
drie.response_message)
raise ApiError(
title="Document(s) invalid syntax or otherwise unsuitable",
description=drie.response_message,
)
description=drie.response_message)
# reset the revision dict so it regenerates.
self.revision_dict = None
return self._get_buffer_rev_id()

9
shipyard_airflow/policy.py
View File

@ -98,6 +98,15 @@ class ShipyardPolicy(object):
'method': 'POST'
}]
),
policy.DocumentedRuleDefault(
'workflow_orchestrator:get_configdocs_status',
RULE_ADMIN_REQUIRED,
'Retrieve the status of the configdocs',
[{
'path': '/api/v1.0/configdocs',
'method': 'GET'
}]
),
policy.DocumentedRuleDefault(
'workflow_orchestrator:create_configdocs',
RULE_ADMIN_REQUIRED,

255
tests/unit/control/test_configdocs_api.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
""" Tests for the configdocs_api"""
import mock
from mock import patch
import pytest
@ -23,138 +24,158 @@ from shipyard_airflow.control.configdocs.configdocs_api import (
)
from shipyard_airflow.control.configdocs.configdocs_helper import \
ConfigdocsHelper
from shipyard_airflow.control.api_lock import ApiLock
from shipyard_airflow.errors import ApiError
from tests.unit.control import common
CTX = ShipyardRequestContext()
def test__validate_version_parameter():
"""
test of the version parameter validation
"""
cdr = ConfigDocsResource()
with pytest.raises(ApiError):
cdr._validate_version_parameter('asdfjkl')
for version in ('buffer', 'committed'):
try:
cdr._validate_version_parameter(version)
except ApiError:
# should not raise an exception.
assert False
class TestConfigDocsStatusResource():
@patch.object(ConfigdocsHelper, 'get_configdocs_status',
common.str_responder)
def test_on_get(self, api_client):
"""Validate the on_get method returns 200 on success"""
result = api_client.simulate_get(
"/api/v1.0/configdocs", headers=common.AUTH_HEADERS)
assert result.status_code == 200
assert result.text == common.str_responder()
assert result.headers[
'content-type'] == 'application/json; charset=UTF-8'
def test_get_collection():
helper = None
with patch.object(ConfigdocsHelper, 'get_collection_docs') as mock_method:
class TestConfigDocsResource():
@patch.object(ConfigDocsResource, 'post_collection', common.str_responder)
@mock.patch.object(ApiLock, 'release')
@mock.patch.object(ApiLock, 'acquire')
def test_on_post(self, mock_acquire, mock_release, api_client):
result = api_client.simulate_post(
"/api/v1.0/configdocs/coll1", headers=common.AUTH_HEADERS)
assert result.status_code == 201
@patch.object(ConfigDocsResource, 'get_collection', common.str_responder)
def test_configdocs_on_get(self, api_client):
"""Validate the on_get method returns 200 on success"""
result = api_client.simulate_get("/api/v1.0/configdocs/coll1",
headers=common.AUTH_HEADERS)
assert result.status_code == 200
def test__validate_version_parameter(self):
"""
test of the version parameter validation
"""
cdr = ConfigDocsResource()
helper = ConfigdocsHelper(CTX)
cdr.get_collection(helper, 'apples')
with pytest.raises(ApiError):
cdr._validate_version_parameter('asdfjkl')
mock_method.assert_called_once_with('buffer', 'apples')
for version in ('buffer', 'committed'):
try:
cdr._validate_version_parameter(version)
except ApiError:
# should not raise an exception.
assert False
def test_get_collection(self):
helper = None
with patch.object(
ConfigdocsHelper, 'get_collection_docs') as mock_method:
cdr = ConfigDocsResource()
helper = ConfigdocsHelper(CTX)
cdr.get_collection(helper, 'apples')
mock_method.assert_called_once_with('buffer', 'apples')
def test_post_collection(self):
"""
Tests the post collection method of the ConfigdocsResource
"""
helper = None
collection_id = 'trees'
document_data = 'lots of info'
with patch.object(ConfigdocsHelper, 'add_collection') as mock_method:
cdr = ConfigDocsResource()
helper = ConfigdocsHelper(CTX)
helper.is_buffer_valid_for_bucket = lambda a, b: True
helper.get_deckhand_validation_status = (
lambda a: ConfigdocsHelper._format_validations_to_status([], 0)
)
cdr.post_collection(helper=helper,
collection_id=collection_id,
document_data=document_data)
mock_method.assert_called_once_with(collection_id, document_data)
with pytest.raises(ApiError):
cdr = ConfigDocsResource()
helper = ConfigdocsHelper(CTX)
# not valid for bucket
helper.is_buffer_valid_for_bucket = lambda a, b: False
helper.get_deckhand_validation_status = (
lambda a: ConfigdocsHelper._format_validations_to_status([], 0)
)
cdr.post_collection(helper=helper,
collection_id=collection_id,
document_data=document_data)
def test_post_collection():
"""
Tests the post collection method of the ConfigdocsResource
"""
helper = None
collection_id = 'trees'
document_data = 'lots of info'
with patch.object(ConfigdocsHelper, 'add_collection') as mock_method:
cdr = ConfigDocsResource()
helper = ConfigdocsHelper(CTX)
helper.is_buffer_valid_for_bucket = lambda a, b: True
helper.get_deckhand_validation_status = (
lambda a: ConfigdocsHelper._format_validations_to_status([], 0)
)
cdr.post_collection(helper=helper,
collection_id=collection_id,
document_data=document_data)
class TestCommitConfigDocsResource():
def test_commit_configdocs(self):
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_buffer = lambda: {'status': 'Success'}
commit_resp = ccdr.commit_configdocs(helper, False)
mock_method.assert_called_once_with(collection_id, document_data)
mock_method.assert_called_once_with('committed')
assert commit_resp['status'] == 'Success'
with pytest.raises(ApiError):
cdr = ConfigDocsResource()
helper = ConfigdocsHelper(CTX)
# not valid for bucket
helper.is_buffer_valid_for_bucket = lambda a, b: False
helper.get_deckhand_validation_status = (
lambda a: ConfigdocsHelper._format_validations_to_status([], 0)
)
cdr.post_collection(helper=helper,
collection_id=collection_id,
document_data=document_data)
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_buffer = (
lambda: {
'status': 'Failure',
'code': '400 Bad Request',
'message': 'this is a mock response'
}
)
commit_resp = ccdr.commit_configdocs(helper, False)
assert '400' in commit_resp['code']
assert commit_resp['message'] is not None
assert commit_resp['status'] == 'Failure'
def test_commit_configdocs_force(self):
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_buffer = lambda: {'status': 'Failure'}
commit_resp = ccdr.commit_configdocs(helper, True)
def test_commit_configdocs():
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_buffer = lambda: {'status': 'Success'}
commit_resp = ccdr.commit_configdocs(helper, False)
mock_method.assert_called_once_with('committed')
print(commit_resp)
assert '200' in commit_resp['code']
assert 'FORCED' in commit_resp['message']
assert commit_resp['status'] == 'Failure'
mock_method.assert_called_once_with('committed')
assert commit_resp['status'] == 'Success'
def test_commit_configdocs_buffer_err(self):
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_buffer = (
lambda: {
'status': 'Failure',
'code': '400 Bad Request',
'message': 'this is a mock response'
}
)
commit_resp = ccdr.commit_configdocs(helper, False)
assert '400' in commit_resp['code']
assert commit_resp['message'] is not None
assert commit_resp['status'] == 'Failure'
def test_commit_configdocs_force():
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_buffer = lambda: {'status': 'Failure'}
commit_resp = ccdr.commit_configdocs(helper, True)
mock_method.assert_called_once_with('committed')
print(commit_resp)
assert '200' in commit_resp['code']
assert 'FORCED' in commit_resp['message']
assert commit_resp['status'] == 'Failure'
def test_commit_configdocs_buffer_err():
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
with pytest.raises(ApiError):
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: True
helper.get_validations_for_buffer = lambda: {'status': 'Success'}
ccdr.commit_configdocs(helper, False)
@patch.object(ConfigDocsResource, 'get_collection', common.str_responder)
def test_configdocs_on_get(api_client):
"""Validate the on_get method returns 200 on success"""
result = api_client.simulate_get("/api/v1.0/configdocs/coll1",
headers=common.AUTH_HEADERS)
assert result.status_code == 200
with pytest.raises(ApiError):
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: True
helper.get_validations_for_buffer = lambda: {'status': 'Success'}
ccdr.commit_configdocs(helper, False)

296
tests/unit/control/test_configdocs_helper.py
View File

@ -21,88 +21,89 @@ from .fake_response import FakeResponse
from shipyard_airflow.control.base import ShipyardRequestContext
from shipyard_airflow.control.configdocs import configdocs_helper
from shipyard_airflow.control.configdocs.configdocs_helper import (
BufferMode,
ConfigdocsHelper
)
BufferMode, ConfigdocsHelper)
from shipyard_airflow.control.configdocs.deckhand_client import (
DeckhandClient,
DeckhandPaths,
DeckhandResponseError,
NoRevisionsExistError
)
DeckhandClient, DeckhandPaths, DeckhandResponseError,
NoRevisionsExistError)
from shipyard_airflow.errors import ApiError, AppError
CTX = ShipyardRequestContext()
REV_BUFFER_DICT = {
'committed': {'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop', 'slop'],
'tags': ['committed'],
'validationPolicies': {}},
'buffer': {'id': 5,
'url': 'url5',
'createdAt': '2017-07-16T21:23Z',
'buckets': ['mop', 'chum'],
'tags': ['deckhand_sez_hi'],
'validationPolicies': {}},
'latest': {'id': 5,
'url': 'url5',
'createdAt': '2017-07-16T21:23Z',
'buckets': ['mop', 'chum'],
'tags': ['deckhand_sez_hi'],
'validationPolicies': {}},
'committed': {
'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop', 'slop'],
'tags': ['committed'],
'validationPolicies': {}
},
'buffer': {
'id': 5,
'url': 'url5',
'createdAt': '2017-07-16T21:23Z',
'buckets': ['mop', 'chum'],
'tags': ['deckhand_sez_hi'],
'validationPolicies': {}
},
'latest': {
'id': 5,
'url': 'url5',
'createdAt': '2017-07-16T21:23Z',
'buckets': ['mop', 'chum'],
'tags': ['deckhand_sez_hi'],
'validationPolicies': {}
},
'revision_count': 5
}
DIFF_BUFFER_DICT = {
'mop': 'unmodified',
'chum': 'created',
'slop': 'deleted'
}
DIFF_BUFFER_DICT = {'mop': 'unmodified', 'chum': 'created', 'slop': 'deleted'}
REV_BUFF_EMPTY_DICT = {
'committed': {'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': ['committed'],
'validationPolicies': {}},
'committed': {
'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': ['committed'],
'validationPolicies': {}
},
'buffer': None,
'latest': {'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': ['committed'],
'validationPolicies': {}},
'latest': {
'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': ['committed'],
'validationPolicies': {}
},
'revision_count': 3
}
DIFF_BUFF_EMPTY_DICT = {
'mop': 'unmodified'
}
DIFF_BUFF_EMPTY_DICT = {'mop': 'unmodified'}
REV_NO_COMMIT_DICT = {
'committed': None,
'buffer': {'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': [],
'validationPolicies': {}},
'latest': {'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': [],
'validationPolicies': {}},
'buffer': {
'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': [],
'validationPolicies': {}
},
'latest': {
'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': [],
'validationPolicies': {}
},
'revision_count': 3
}
DIFF_NO_COMMIT_DICT = {
'mop': 'created'
}
DIFF_NO_COMMIT_DICT = {'mop': 'created'}
REV_EMPTY_DICT = {
'committed': None,
@ -114,30 +115,34 @@ REV_EMPTY_DICT = {
DIFF_EMPTY_DICT = {}
REV_COMMIT_AND_BUFFER_DICT = {
'committed': {'id': 1,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': ['committed'],
'validationPolicies': {}},
'buffer': {'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': [],
'validationPolicies': {}},
'latest': {'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': [],
'validationPolicies': {}},
'committed': {
'id': 1,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': ['committed'],
'validationPolicies': {}
},
'buffer': {
'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': [],
'validationPolicies': {}
},
'latest': {
'id': 3,
'url': 'url3',
'createdAt': '2017-07-14T21:23Z',
'buckets': ['mop'],
'tags': [],
'validationPolicies': {}
},
'revision_count': 3
}
DIFF_COMMIT_AND_BUFFER_DICT = {
'mop': 'modified'
}
DIFF_COMMIT_AND_BUFFER_DICT = {'mop': 'modified'}
def test_construct_configdocs_helper():
@ -154,33 +159,22 @@ def test_get_buffer_mode():
ensures that strings passed to get_buffer_mode are properly handled
"""
# None cases
assert ConfigdocsHelper.get_buffer_mode('') == BufferMode.REJECTONCONTENTS
assert ConfigdocsHelper.get_buffer_mode(
''
) == BufferMode.REJECTONCONTENTS
assert ConfigdocsHelper.get_buffer_mode(
None
) == BufferMode.REJECTONCONTENTS
None) == BufferMode.REJECTONCONTENTS
# valid cases
assert ConfigdocsHelper.get_buffer_mode(
'rejectoncontents'
) == BufferMode.REJECTONCONTENTS
assert ConfigdocsHelper.get_buffer_mode(
'append'
) == BufferMode.APPEND
assert ConfigdocsHelper.get_buffer_mode(
'replace'
) == BufferMode.REPLACE
'rejectoncontents') == BufferMode.REJECTONCONTENTS
assert ConfigdocsHelper.get_buffer_mode('append') == BufferMode.APPEND
assert ConfigdocsHelper.get_buffer_mode('replace') == BufferMode.REPLACE
# case insensitive
assert ConfigdocsHelper.get_buffer_mode(
'ReJEcTOnConTenTs'
) == BufferMode.REJECTONCONTENTS
'ReJEcTOnConTenTs') == BufferMode.REJECTONCONTENTS
# bad value
assert ConfigdocsHelper.get_buffer_mode(
'hippopotomus'
) is None
assert ConfigdocsHelper.get_buffer_mode('hippopotomus') is None
def test_is_buffer_emtpy():
@ -208,8 +202,7 @@ def test_is_collection_in_buffer():
helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_BUFFER_DICT
)
lambda old_revision_id, new_revision_id: DIFF_BUFFER_DICT)
# mop is not in buffer; chum and slop are in buffer.
# unmodified means it is not in buffer
assert not helper.is_collection_in_buffer('mop')
@ -220,9 +213,7 @@ def test_is_collection_in_buffer():
def _raise_dre():
raise DeckhandResponseError(
status_code=9000,
response_message='This is bogus'
)
status_code=9000, response_message='This is bogus')
helper._get_revision_dict = _raise_dre
@ -242,8 +233,7 @@ def test_is_buffer_valid_for_bucket():
helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_BUFFER_DICT
)
lambda old_revision_id, new_revision_id: DIFF_BUFFER_DICT)
helper.deckhand.rollback = lambda target_revision_id: (
set_revision_dict(helper, REV_BUFF_EMPTY_DICT, DIFF_BUFF_EMPTY_DICT)
)
@ -279,8 +269,7 @@ def test_is_buffer_valid_for_bucket():
# set up as if there is no committed revision yet
helper._get_revision_dict = lambda: REV_NO_COMMIT_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_NO_COMMIT_DICT
)
lambda old_revision_id, new_revision_id: DIFF_NO_COMMIT_DICT)
assert helper.is_buffer_valid_for_bucket('slop', BufferMode.APPEND)
assert helper.is_buffer_valid_for_bucket('chum', BufferMode.APPEND)
@ -295,8 +284,7 @@ def test_is_buffer_valid_for_bucket():
# set up as if there is nothing in deckhand.
helper._get_revision_dict = lambda: REV_EMPTY_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_EMPTY_DICT
)
lambda old_revision_id, new_revision_id: DIFF_EMPTY_DICT)
# should be able to add in any mode.
assert helper.is_buffer_valid_for_bucket('slop', BufferMode.APPEND)
assert helper.is_buffer_valid_for_bucket('chum', BufferMode.APPEND)
@ -305,6 +293,30 @@ def test_is_buffer_valid_for_bucket():
assert helper.is_buffer_valid_for_bucket('mop', BufferMode.REPLACE)
def test_get_configdocs_status():
helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_BUFFER_DICT)
result = helper.get_configdocs_status()
expected = [{
"collection_name": 'chum',
"committed_status": 'not present',
"buffer_status": 'created'
}, {
"collection_name": 'mop',
"committed_status": 'present',
"buffer_status": 'unmodified'
}, {
"collection_name": 'slop',
"committed_status": 'present',
"buffer_status": 'deleted'
}]
assert expected == sorted(result, key=lambda x: x['collection_name'])
def test__get_revision_dict_no_commit():
"""
Tests the processing of revision dict response from dechand
@ -439,11 +451,10 @@ def test__get_revision_dict_errs():
tests getting a revision dictionary method when the deckhand
client has raised an exception
"""
def _raise_dre():
raise DeckhandResponseError(
status_code=9000,
response_message='This is bogus'
)
status_code=9000, response_message='This is bogus')
def _raise_nree():
raise NoRevisionsExistError()
@ -472,12 +483,10 @@ def test_get_collection_docs():
"""
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_docs_from_revision = (
lambda revision_id, bucket_id: "{'yaml': 'yaml'}"
)
lambda revision_id, bucket_id: "{'yaml': 'yaml'}")
helper._get_revision_dict = lambda: REV_EMPTY_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_EMPTY_DICT
)
lambda old_revision_id, new_revision_id: DIFF_EMPTY_DICT)
with pytest.raises(ApiError):
helper.get_collection_docs(configdocs_helper.BUFFER, 'mop')
@ -487,31 +496,30 @@ def test_get_collection_docs():
helper._get_revision_dict = lambda: REV_COMMIT_AND_BUFFER_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_COMMIT_AND_BUFFER_DICT
)
lambda old_revision_id, new_revision_id: DIFF_COMMIT_AND_BUFFER_DICT)
yaml_str = helper.get_collection_docs(configdocs_helper.BUFFER, 'mop')
print(yaml_str)
assert len(yaml_str) == 16
yaml_str = helper.get_collection_docs(configdocs_helper.COMMITTED, 'mop')
print(yaml_str)
assert len(yaml_str) == 16
def _fake_get_validation_endpoints():
val_ep = '{}/validatedesign'
return [
{'name': 'Drydock', 'url': val_ep.format('drydock')},
{'name': 'Armada', 'url': val_ep.format('armada')},
{
'name': 'Drydock',
'url': val_ep.format('drydock')
},
{
'name': 'Armada',
'url': val_ep.format('armada')
},
]
def _fake_get_validations_for_component(url,
design_reference,
response,
exception,
context_marker,
**kwargs):
def _fake_get_validations_for_component(url, design_reference, response,
exception, context_marker, **kwargs):
"""
Responds with a status response
"""
@ -547,11 +555,9 @@ def test_get_validations_for_revision():
hold_ve = helper.__class__._get_validation_endpoints
hold_vfc = helper.__class__._get_validations_for_component
helper.__class__._get_validation_endpoints = (
_fake_get_validation_endpoints
)
_fake_get_validation_endpoints)
helper.__class__._get_validations_for_component = (
_fake_get_validations_for_component
)
_fake_get_validations_for_component)
helper._get_deckhand_validations = lambda revision_id: []
try:
val_status = helper.get_validations_for_revision(3)
@ -565,7 +571,9 @@ def test_get_validations_for_revision():
mock_get_path.assert_called_with(DeckhandPaths.RENDERED_REVISION_DOCS)
FK_VAL_BASE_RESP = FakeResponse(status_code=200, text="""
FK_VAL_BASE_RESP = FakeResponse(
status_code=200,
text="""
---
count: 2
next: null
@ -580,7 +588,9 @@ results:
...
""")
FK_VAL_SUBSET_RESP = FakeResponse(status_code=200, text="""
FK_VAL_SUBSET_RESP = FakeResponse(
status_code=200,
text="""
---
count: 1
next: null
@ -592,8 +602,9 @@ results:
...
""")
FK_VAL_ENTRY_RESP = FakeResponse(status_code=200, text="""
FK_VAL_ENTRY_RESP = FakeResponse(
status_code=200,
text="""
---
name: promenade-site-validation
url: https://deckhand/a/url/too/long/for/pep8
@ -618,14 +629,11 @@ def test__get_deckhand_validations():
"""
helper = ConfigdocsHelper(CTX)
helper.deckhand._get_base_validation_resp = (
lambda revision_id: FK_VAL_BASE_RESP
)
lambda revision_id: FK_VAL_BASE_RESP)
helper.deckhand._get_subset_validation_response = (
lambda reivsion_id, subset_name: FK_VAL_SUBSET_RESP
)
lambda reivsion_id, subset_name: FK_VAL_SUBSET_RESP)
helper.deckhand._get_entry_validation_response = (
lambda reivsion_id, subset_name, entry_id: FK_VAL_ENTRY_RESP
)
lambda reivsion_id, subset_name, entry_id: FK_VAL_ENTRY_RESP)
assert len(helper._get_deckhand_validations(5)) == 2