summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAaron Sheffield <ajs@sheffieldfamily.net>2018-10-12 09:00:32 -0500
committerAaron Sheffield <ajs@sheffieldfamily.net>2018-10-23 11:12:56 -0500
commitd62f15123dafa5b77f1bbcedb99b21c2d81023b2 (patch)
treedcfbc8fa82dd251704f43aa2f58edb2240574eea
parentf5774206e502ddee61a029cad380bb1f4350417e (diff)
Expects Redacted Raw Documents
- Adds a query parameter 'cleartext-secrets' to get full raw documents. - Adds CLI flag to get full raw documents. Change-Id: If38974c8433c8360cc47ae1273720ad76e87a6fd
Notes
Notes (review): Code-Review+2: Scott Hussey <sthussey@att.com> Code-Review+2: Felipe Monteiro <felipe.monteiro@att.com> Code-Review+1: Nishant Kumar <nishant.e.kumar@ericsson.com> Workflow+1: Bryan Strassner <bryan.strassner@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Tue, 06 Nov 2018 21:57:46 +0000 Reviewed-on: https://review.openstack.org/610042 Project: openstack/airship-shipyard Branch: refs/heads/master
-rw-r--r--doc/source/API.rst5
-rw-r--r--doc/source/CLI.rst5
-rw-r--r--src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py10
-rw-r--r--src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py20
-rw-r--r--src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py12
-rw-r--r--src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py2
-rw-r--r--src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py2
-rw-r--r--src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py5
-rw-r--r--src/bin/shipyard_client/shipyard_client/cli/get/actions.py6
-rw-r--r--src/bin/shipyard_client/shipyard_client/cli/get/commands.py10
-rw-r--r--src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py2
11 files changed, 58 insertions, 21 deletions
diff --git a/doc/source/API.rst b/doc/source/API.rst
index 130f15b..79bc8ea 100644
--- a/doc/source/API.rst
+++ b/doc/source/API.rst
@@ -220,8 +220,11 @@ Returns the source documents for a collection of documents
220 220
221Query Parameters 221Query Parameters
222'''''''''''''''' 222''''''''''''''''
223version=committed | last_site_action | successful_site_action | **buffer** 223- version=committed | last_site_action | successful_site_action | **buffer**
224 Return the documents for the version specified - buffer by default. 224 Return the documents for the version specified - buffer by default.
225- cleartext-secrets=true/**false**
226 If true then returns cleartext secrets in encrypted documents, otherwise
227 those values are redacted.
225 228
226Responses 229Responses
227''''''''' 230'''''''''
diff --git a/doc/source/CLI.rst b/doc/source/CLI.rst
index b166503..2318c79 100644
--- a/doc/source/CLI.rst
+++ b/doc/source/CLI.rst
@@ -647,6 +647,7 @@ differences between the 'committed' and 'buffer' revision (default behavior).
647 shipyard get configdocs 647 shipyard get configdocs
648 [--collection=<collection>] 648 [--collection=<collection>]
649 [--committed | --last-site-action | --successful-site-action | --buffer] 649 [--committed | --last-site-action | --successful-site-action | --buffer]
650 [--cleartext-secrets]
650 651
651 Example: 652 Example:
652 shipyard get configdocs --collection=design 653 shipyard get configdocs --collection=design
@@ -675,6 +676,10 @@ differences between the 'committed' and 'buffer' revision (default behavior).
675 prior commit. If no documents have been loaded into the buffer for this 676 prior commit. If no documents have been loaded into the buffer for this
676 collection, this will return an empty response (default) 677 collection, this will return an empty response (default)
677 678
679\--cleartext-secrets
680 Returns cleartext secrets in encrypted documents, otherwise those values
681 are redacted. Only impacts returned documents, not lists of documents.
682
678Sample 683Sample
679^^^^^^ 684^^^^^^
680 685
diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py
index 813dcfe..a778c98 100644
--- a/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py
+++ b/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py
@@ -98,11 +98,13 @@ class ConfigDocsResource(BaseResource):
98 Returns a collection of documents 98 Returns a collection of documents
99 """ 99 """
100 version = (req.params.get('version') or 'buffer') 100 version = (req.params.get('version') or 'buffer')
101 cleartext_secrets = req.get_param_as_bool('cleartext-secrets')
101 self._validate_version_parameter(version) 102 self._validate_version_parameter(version)
102 helper = ConfigdocsHelper(req.context) 103 helper = ConfigdocsHelper(req.context)
103 # Not reformatting to JSON or YAML since just passing through 104 # Not reformatting to JSON or YAML since just passing through
104 resp.body = self.get_collection( 105 resp.body = self.get_collection(
105 helper=helper, collection_id=collection_id, version=version) 106 helper=helper, collection_id=collection_id, version=version,
107 cleartext_secrets=cleartext_secrets)
106 resp.append_header('Content-Type', 'application/x-yaml') 108 resp.append_header('Content-Type', 'application/x-yaml')
107 resp.status = falcon.HTTP_200 109 resp.status = falcon.HTTP_200
108 110
@@ -116,13 +118,15 @@ class ConfigDocsResource(BaseResource):
116 status=falcon.HTTP_400, 118 status=falcon.HTTP_400,
117 retry=False, ) 119 retry=False, )
118 120
119 def get_collection(self, helper, collection_id, version='buffer'): 121 def get_collection(self, helper, collection_id, version='buffer',
122 cleartext_secrets=False):
120 """ 123 """
121 Attempts to retrieve the specified collection of documents 124 Attempts to retrieve the specified collection of documents
122 either from the buffer, committed version, last site action 125 either from the buffer, committed version, last site action
123 or successful site action, as specified 126 or successful site action, as specified
124 """ 127 """
125 return helper.get_collection_docs(version, collection_id) 128 return helper.get_collection_docs(version, collection_id,
129 cleartext_secrets)
126 130
127 def post_collection(self, 131 def post_collection(self,
128 helper, 132 helper,
diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py
index df6850a..3ba2722 100644
--- a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py
+++ b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py
@@ -318,18 +318,21 @@ class ConfigdocsHelper(object):
318 rev = self._get_revision_dict().get(target_revision) 318 rev = self._get_revision_dict().get(target_revision)
319 return rev['id'] if rev else None 319 return rev['id'] if rev else None
320 320
321 def get_collection_docs(self, version, collection_id): 321 def get_collection_docs(self, version, collection_id,
322 cleartext_secrets=False):
322 """ 323 """
323 Returns the requested collection of docs based on the version 324 Returns the requested collection of docs based on the version
324 specifier. The default is set as buffer. 325 specifier. The default is set as buffer.
325 """ 326 """
326 LOG.info('Retrieving collection %s from %s', collection_id, version) 327 LOG.info('Retrieving collection %s from %s', collection_id, version)
327 if version in [COMMITTED, LAST_SITE_ACTION, SUCCESSFUL_SITE_ACTION]: 328 if version in [COMMITTED, LAST_SITE_ACTION, SUCCESSFUL_SITE_ACTION]:
328 return self._get_target_docs(collection_id, version) 329 return self._get_target_docs(collection_id, version,
330 cleartext_secrets)
329 331
330 return self._get_doc_from_buffer(collection_id) 332 return self._get_doc_from_buffer(collection_id,
333 cleartext_secrets=cleartext_secrets)
331 334
332 def _get_doc_from_buffer(self, collection_id): 335 def _get_doc_from_buffer(self, collection_id, cleartext_secrets=False):
333 """ 336 """
334 Returns the collection if it exists in the buffer. 337 Returns the collection if it exists in the buffer.
335 If the buffer contains the collection, the latest 338 If the buffer contains the collection, the latest
@@ -343,7 +346,8 @@ class ConfigdocsHelper(object):
343 # revision exists 346 # revision exists
344 buffer_id = self.get_revision_id(BUFFER) 347 buffer_id = self.get_revision_id(BUFFER)
345 return self.deckhand.get_docs_from_revision( 348 return self.deckhand.get_docs_from_revision(
346 revision_id=buffer_id, bucket_id=collection_id) 349 revision_id=buffer_id, bucket_id=collection_id,
350 cleartext_secrets=cleartext_secrets)
347 raise ApiError( 351 raise ApiError(
348 title='No documents to retrieve', 352 title='No documents to retrieve',
349 description=('The Shipyard buffer is empty or does not contain ' 353 description=('The Shipyard buffer is empty or does not contain '
@@ -351,7 +355,8 @@ class ConfigdocsHelper(object):
351 status=falcon.HTTP_404, 355 status=falcon.HTTP_404,
352 retry=False) 356 retry=False)
353 357
354 def _get_target_docs(self, collection_id, target_rev): 358 def _get_target_docs(self, collection_id, target_rev,
359 cleartext_secrets=False):
355 """ 360 """
356 Returns the collection if it exists as committed, last_site_action 361 Returns the collection if it exists as committed, last_site_action
357 or successful_site_action. 362 or successful_site_action.
@@ -360,7 +365,8 @@ class ConfigdocsHelper(object):
360 365
361 if revision_id: 366 if revision_id:
362 return self.deckhand.get_docs_from_revision( 367 return self.deckhand.get_docs_from_revision(
363 revision_id=revision_id, bucket_id=collection_id) 368 revision_id=revision_id, bucket_id=collection_id,
369 cleartext_secrets=cleartext_secrets)
364 370
365 raise ApiError( 371 raise ApiError(
366 title='No documents to retrieve', 372 title='No documents to retrieve',
diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py
index 9c0d462..4b43028 100644
--- a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py
+++ b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/deckhand_client.py
@@ -184,20 +184,25 @@ class DeckhandClient(object):
184 diff = yaml.safe_load(response.text) 184 diff = yaml.safe_load(response.text)
185 return diff 185 return diff
186 186
187 def get_docs_from_revision(self, revision_id, bucket_id=None): 187 def get_docs_from_revision(self, revision_id, bucket_id=None,
188 cleartext_secrets=False):
188 """ 189 """
189 Retrieves the collection of docs from the revision specified 190 Retrieves the collection of docs from the revision specified
190 for the bucket_id specified 191 for the bucket_id specified
192 cleartext_secrets: Should deckhand show or redact secrets.
191 :returns: a string representing the response.text from Deckhand 193 :returns: a string representing the response.text from Deckhand
192 """ 194 """
195
193 url = DeckhandClient.get_path( 196 url = DeckhandClient.get_path(
194 DeckhandPaths.REVISION_DOCS 197 DeckhandPaths.REVISION_DOCS
195 ).format(revision_id) 198 ).format(revision_id)
196 199
197 # if a bucket_id is specified, limit the response to a bucket 200 # if a bucket_id is specified, limit the response to a bucket
198 query = None 201 query = {}
199 if bucket_id is not None: 202 if bucket_id is not None:
200 query = {'status.bucket': bucket_id} 203 query = {'status.bucket': bucket_id}
204 if cleartext_secrets is True:
205 query['cleartext-secrets'] = 'true'
201 response = self._get_request(url, params=query) 206 response = self._get_request(url, params=query)
202 self._handle_bad_response(response) 207 self._handle_bad_response(response)
203 return response.text 208 return response.text
@@ -336,6 +341,9 @@ class DeckhandClient(object):
336 'X-Auth-Token': get_token() 341 'X-Auth-Token': get_token()
337 } 342 }
338 343
344 if not params:
345 params = None
346
339 DeckhandClient._log_request('GET', url, params) 347 DeckhandClient._log_request('GET', url, params)
340 response = requests.get( 348 response = requests.get(
341 url, 349 url,
diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py b/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py
index 7704dc5..969d417 100644
--- a/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py
+++ b/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py
@@ -96,7 +96,7 @@ class TestConfigDocsResource():
96 helper = ConfigdocsHelper(CTX) 96 helper = ConfigdocsHelper(CTX)
97 cdr.get_collection(helper, 'apples') 97 cdr.get_collection(helper, 'apples')
98 98
99 mock_method.assert_called_once_with('buffer', 'apples') 99 mock_method.assert_called_once_with('buffer', 'apples', False)
100 100
101 @patch.object(ConfigdocsHelper, 'is_collection_in_buffer', 101 @patch.object(ConfigdocsHelper, 'is_collection_in_buffer',
102 lambda x, y: True) 102 lambda x, y: True)
diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py b/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py
index b34b73f..3a5cc5a 100644
--- a/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py
+++ b/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py
@@ -503,7 +503,7 @@ def test_get_collection_docs():
503 """ 503 """
504 helper = ConfigdocsHelper(CTX) 504 helper = ConfigdocsHelper(CTX)
505 helper.deckhand.get_docs_from_revision = ( 505 helper.deckhand.get_docs_from_revision = (
506 lambda revision_id, bucket_id: "{'yaml': 'yaml'}") 506 lambda revision_id, bucket_id, cleartext_secrets: "{'yaml': 'yaml'}")
507 helper._get_revision_dict = lambda: REV_EMPTY_DICT 507 helper._get_revision_dict = lambda: REV_EMPTY_DICT
508 helper.deckhand.get_diff = ( 508 helper.deckhand.get_diff = (
509 lambda old_revision_id, new_revision_id: DIFF_EMPTY_DICT) 509 lambda old_revision_id, new_revision_id: DIFF_EMPTY_DICT)
diff --git a/src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py b/src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py
index d91b06c..42b578f 100644
--- a/src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py
+++ b/src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py
@@ -68,7 +68,8 @@ class ShipyardClient(BaseClient):
68 ) 68 )
69 return self.post_resp(url, query_params, document_data) 69 return self.post_resp(url, query_params, document_data)
70 70
71 def get_configdocs(self, collection_id=None, version='buffer'): 71 def get_configdocs(self, collection_id=None, version='buffer',
72 cleartext_secrets=False):
72 """ 73 """
73 Get the collection of documents from deckhand specified by 74 Get the collection of documents from deckhand specified by
74 collection id 75 collection id
@@ -78,6 +79,8 @@ class ShipyardClient(BaseClient):
78 :rtype: Response object 79 :rtype: Response object
79 """ 80 """
80 query_params = {"version": version} 81 query_params = {"version": version}
82 if cleartext_secrets is True:
83 query_params['cleartext-secrets'] = 'true'
81 url = ApiPaths.POST_GET_CONFIG.value.format( 84 url = ApiPaths.POST_GET_CONFIG.value.format(
82 self.get_endpoint(), 85 self.get_endpoint(),
83 collection_id) 86 collection_id)
diff --git a/src/bin/shipyard_client/shipyard_client/cli/get/actions.py b/src/bin/shipyard_client/shipyard_client/cli/get/actions.py
index 27bc4f8..b839756 100644
--- a/src/bin/shipyard_client/shipyard_client/cli/get/actions.py
+++ b/src/bin/shipyard_client/shipyard_client/cli/get/actions.py
@@ -48,7 +48,7 @@ class GetActions(CliAction):
48class GetConfigdocs(CliAction): 48class GetConfigdocs(CliAction):
49 """Action to Get Configdocs""" 49 """Action to Get Configdocs"""
50 50
51 def __init__(self, ctx, collection, version): 51 def __init__(self, ctx, collection, version, cleartext_secrets=False):
52 """Sets parameters.""" 52 """Sets parameters."""
53 super().__init__(ctx) 53 super().__init__(ctx)
54 self.logger.debug( 54 self.logger.debug(
@@ -56,12 +56,14 @@ class GetConfigdocs(CliAction):
56 "version=%s" % (collection, version)) 56 "version=%s" % (collection, version))
57 self.collection = collection 57 self.collection = collection
58 self.version = version 58 self.version = version
59 self.cleartext_secrets = cleartext_secrets
59 60
60 def invoke(self): 61 def invoke(self):
61 """Calls API Client and formats response from API Client""" 62 """Calls API Client and formats response from API Client"""
62 self.logger.debug("Calling API Client get_configdocs.") 63 self.logger.debug("Calling API Client get_configdocs.")
63 return self.get_api_client().get_configdocs( 64 return self.get_api_client().get_configdocs(
64 collection_id=self.collection, version=self.version) 65 collection_id=self.collection, version=self.version,
66 cleartext_secrets=self.cleartext_secrets)
65 67
66 # Handle 404 with default error handler for cli. 68 # Handle 404 with default error handler for cli.
67 cli_handled_err_resp_codes = [404] 69 cli_handled_err_resp_codes = [404]
diff --git a/src/bin/shipyard_client/shipyard_client/cli/get/commands.py b/src/bin/shipyard_client/shipyard_client/cli/get/commands.py
index fdc6db4..48adc47 100644
--- a/src/bin/shipyard_client/shipyard_client/cli/get/commands.py
+++ b/src/bin/shipyard_client/shipyard_client/cli/get/commands.py
@@ -98,16 +98,22 @@ SHORT_DESC_CONFIGDOCS = ("Retrieve documents loaded into Shipyard, either "
98 flag_value='successful_site_action', 98 flag_value='successful_site_action',
99 help='Holds the revision information for the most recent successfully ' 99 help='Holds the revision information for the most recent successfully '
100 'executed site action.') 100 'executed site action.')
101@click.option(
102 '--cleartext-secrets',
103 '-t',
104 help='Returns cleartext secrets in documents',
105 is_flag=True)
101@click.pass_context 106@click.pass_context
102def get_configdocs(ctx, collection, buffer, committed, last_site_action, 107def get_configdocs(ctx, collection, buffer, committed, last_site_action,
103 successful_site_action): 108 successful_site_action, cleartext_secrets):
104 if collection: 109 if collection:
105 # Get version 110 # Get version
106 _version = get_version(ctx, buffer, committed, last_site_action, 111 _version = get_version(ctx, buffer, committed, last_site_action,
107 successful_site_action) 112 successful_site_action)
108 113
109 click.echo( 114 click.echo(
110 GetConfigdocs(ctx, collection, _version).invoke_and_return_resp()) 115 GetConfigdocs(ctx, collection, _version,
116 cleartext_secrets).invoke_and_return_resp())
111 117
112 else: 118 else:
113 compare_versions = check_reformat_versions(ctx, 119 compare_versions = check_reformat_versions(ctx,
diff --git a/src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py b/src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py
index 17e91ea..25bf411 100644
--- a/src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py
+++ b/src/bin/shipyard_client/tests/unit/cli/get/test_get_commands.py
@@ -58,7 +58,7 @@ def test_get_configdocs_with_passing_collection(*args):
58 with patch.object(GetConfigdocs, '__init__') as mock_method: 58 with patch.object(GetConfigdocs, '__init__') as mock_method:
59 runner.invoke(shipyard, [auth_vars, 'get', 'configdocs', 59 runner.invoke(shipyard, [auth_vars, 'get', 'configdocs',
60 '--collection=design']) 60 '--collection=design'])
61 mock_method.assert_called_once_with(ANY, 'design', 'buffer') 61 mock_method.assert_called_once_with(ANY, 'design', 'buffer', False)
62 62
63 63
64def test_get_configdocs_without_passing_collection(*args): 64def test_get_configdocs_without_passing_collection(*args):