summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFelipe Monteiro <felipe.monteiro@att.com>2018-10-31 15:27:38 -0400
committerFelipe Monteiro <felipe.monteiro@att.com>2018-11-08 20:07:03 -0500
commitf8d79e119cd17b93f2655876037b35c0f1e55518 (patch)
treee4048a69fced077bb3025afd9718dbefff758f66
parentd7740b0f405c6f90e186f5dc6f943a18cf870299 (diff)
Only collect/parse Deckhand-formatted documents for processing
This patch set changes Pegleg in two similar ways: 1) Ignore certain types of files altogether: - those located in hidden folders - those prefixed with "." (files like .zuul.yaml) 2) Only read Deckhand-formatted documents for lint/collect/etc. commands as Pegleg need not consider other types of documents (it separately reads the site-definition.yaml for internal processing still). The tools/ subfolder is also ignored as it can contain .yaml files which are not Deckhand-formatted documents, so need not be processed by pegleg.engine. Change-Id: I8996b5d430cf893122af648ef8e5805b36c1bfd9
Notes
Notes (review): Code-Review+1: Tin Lam <tin@irrational.io> Code-Review+2: Scott Hussey <sthussey@att.com> Code-Review+1: Ahmad Mahmoudi <am495p@att.com> Code-Review+2: Bryan Strassner <bryan.strassner@gmail.com> Workflow+1: Bryan Strassner <bryan.strassner@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Fri, 09 Nov 2018 20:00:27 +0000 Reviewed-on: https://review.openstack.org/614621 Project: openstack/airship-pegleg Branch: refs/heads/master
-rw-r--r--pegleg/cli.py2
-rw-r--r--pegleg/engine/lint.py21
-rw-r--r--pegleg/engine/util/definition.py8
-rw-r--r--pegleg/engine/util/files.py46
-rw-r--r--pegleg/engine/util/pegleg_secret_management.py8
-rw-r--r--tests/unit/engine/test_encryption.py77
-rw-r--r--tests/unit/engine/test_lint.py17
-rw-r--r--tests/unit/engine/util/test_files.py38
8 files changed, 134 insertions, 83 deletions
diff --git a/pegleg/cli.py b/pegleg/cli.py
index 0e3614d..a7c3b43 100644
--- a/pegleg/cli.py
+++ b/pegleg/cli.py
@@ -385,7 +385,7 @@ def secrets():
385 'author', 385 'author',
386 required=True, 386 required=True,
387 help='Identifier for the program or person who is encrypting the secrets ' 387 help='Identifier for the program or person who is encrypting the secrets '
388 'documents') 388 'documents')
389@click.argument('site_name') 389@click.argument('site_name')
390def encrypt(*, save_location, author, site_name): 390def encrypt(*, save_location, author, site_name):
391 engine.repository.process_repositories(site_name) 391 engine.repository.process_repositories(site_name)
diff --git a/pegleg/engine/lint.py b/pegleg/engine/lint.py
index 550c065..c0706ce 100644
--- a/pegleg/engine/lint.py
+++ b/pegleg/engine/lint.py
@@ -18,7 +18,6 @@ import os
18import pkg_resources 18import pkg_resources
19import shutil 19import shutil
20import textwrap 20import textwrap
21import yaml
22 21
23from prettytable import PrettyTable 22from prettytable import PrettyTable
24 23
@@ -223,16 +222,16 @@ def _verify_single_file(filename, schemas):
223 errors.append((FILE_MISSING_YAML_DOCUMENT_HEADER, 222 errors.append((FILE_MISSING_YAML_DOCUMENT_HEADER,
224 '%s does not begin with YAML beginning of document ' 223 '%s does not begin with YAML beginning of document '
225 'marker "---".' % filename)) 224 'marker "---".' % filename))
226 f.seek(0) 225
227 documents = [] 226 documents = []
228 try: 227 try:
229 documents = list(yaml.safe_load_all(f)) 228 documents = util.files.read(filename)
230 except Exception as e: 229 except Exception as e:
231 errors.append((FILE_CONTAINS_INVALID_YAML, 230 errors.append((FILE_CONTAINS_INVALID_YAML,
232 '%s is not valid yaml: %s' % (filename, e))) 231 '%s is not valid yaml: %s' % (filename, e)))
233 232
234 for document in documents: 233 for document in documents:
235 errors.extend(_verify_document(document, schemas, filename)) 234 errors.extend(_verify_document(document, schemas, filename))
236 235
237 return errors 236 return errors
238 237
diff --git a/pegleg/engine/util/definition.py b/pegleg/engine/util/definition.py
index 117d2a3..1b2c39c 100644
--- a/pegleg/engine/util/definition.py
+++ b/pegleg/engine/util/definition.py
@@ -15,7 +15,6 @@
15import os 15import os
16 16
17import click 17import click
18import yaml
19 18
20from pegleg import config 19from pegleg import config
21from pegleg.engine.util import files 20from pegleg.engine.util import files
@@ -52,6 +51,7 @@ def load_as_params(site_name, *fields, primary_repo_base=None):
52 51
53 52
54def path(site_name, primary_repo_base=None): 53def path(site_name, primary_repo_base=None):
54 """Retrieve path to the site-definition.yaml file for ``site_name``."""
55 if not primary_repo_base: 55 if not primary_repo_base:
56 primary_repo_base = config.get_site_repo() 56 primary_repo_base = config.get_site_repo()
57 return os.path.join(primary_repo_base, 'site', site_name, 57 return os.path.join(primary_repo_base, 'site', site_name,
@@ -100,8 +100,7 @@ def documents_for_each_site():
100 paths = files.directories_for(**params) 100 paths = files.directories_for(**params)
101 filenames = set(files.search(paths)) 101 filenames = set(files.search(paths))
102 for filename in filenames: 102 for filename in filenames:
103 with open(filename) as f: 103 documents[sitename].extend(files.read(filename))
104 documents[sitename].extend(list(yaml.safe_load_all(f)))
105 104
106 return documents 105 return documents
107 106
@@ -122,7 +121,6 @@ def documents_for_site(sitename):
122 paths = files.directories_for(**params) 121 paths = files.directories_for(**params)
123 filenames = set(files.search(paths)) 122 filenames = set(files.search(paths))
124 for filename in filenames: 123 for filename in filenames:
125 with open(filename) as f: 124 documents.extend(files.read(filename))
126 documents.extend(list(yaml.safe_load_all(f)))
127 125
128 return documents 126 return documents
diff --git a/pegleg/engine/util/files.py b/pegleg/engine/util/files.py
index 5281dbb..0ab0cae 100644
--- a/pegleg/engine/util/files.py
+++ b/pegleg/engine/util/files.py
@@ -18,6 +18,7 @@ import yaml
18import logging 18import logging
19 19
20from pegleg import config 20from pegleg import config
21from pegleg.engine.util import pegleg_managed_document as md
21 22
22LOG = logging.getLogger(__name__) 23LOG = logging.getLogger(__name__)
23 24
@@ -248,9 +249,35 @@ def read(path):
248 '{} not found. Pegleg must be run from the root of a ' 249 '{} not found. Pegleg must be run from the root of a '
249 'configuration repository.'.format(path)) 250 'configuration repository.'.format(path))
250 251
252 def is_deckhand_document(document):
253 # Deckhand documents only consist of control and application
254 # documents.
255 valid_schemas = ('metadata/Control', 'metadata/Document')
256 if isinstance(document, dict):
257 schema = document.get('metadata', {}).get('schema', '')
258 # NOTE(felipemonteiro): The Pegleg site-definition.yaml is a
259 # Deckhand-formatted document currently but probably shouldn't
260 # be, because it has no business being in Deckhand. As such,
261 # treat it as a special case.
262 if "SiteDefinition" in document.get('schema', ''):
263 return False
264 if any(schema.startswith(x) for x in valid_schemas):
265 return True
266 else:
267 LOG.debug('Document with schema=%s is not a valid Deckhand '
268 'schema. Ignoring it.', schema)
269 return False
270
271 def is_pegleg_managed_document(document):
272 return md.PeglegManagedSecretsDocument.is_pegleg_managed_secret(
273 document)
274
251 with open(path) as stream: 275 with open(path) as stream:
252 try: 276 try:
253 return list(yaml.safe_load_all(stream)) 277 return [
278 d for d in yaml.safe_load_all(stream)
279 if is_deckhand_document(d) or is_pegleg_managed_document(d)
280 ]
254 except yaml.YAMLError as e: 281 except yaml.YAMLError as e:
255 raise click.ClickException('Failed to parse %s:\n%s' % (path, e)) 282 raise click.ClickException('Failed to parse %s:\n%s' % (path, e))
256 283
@@ -296,10 +323,25 @@ def _recurse_subdirs(search_path, depth):
296 323
297 324
298def search(search_paths): 325def search(search_paths):
326 if not isinstance(search_paths, (list, tuple)):
327 search_paths = [search_paths]
328
299 for search_path in search_paths: 329 for search_path in search_paths:
300 LOG.debug("Recursively collecting YAMLs from %s" % search_path) 330 LOG.debug("Recursively collecting YAMLs from %s" % search_path)
301 for root, _dirs, filenames in os.walk(search_path): 331 for root, _, filenames in os.walk(search_path):
332
333 # Ignore hidden folders like .tox or .git for faster processing.
334 if os.path.basename(root).startswith("."):
335 continue
336 # Skip over anything in tools/ because it will never contain valid
337 # Pegleg-owned manifest documents.
338 if "tools" in root.split("/"):
339 continue
340
302 for filename in filenames: 341 for filename in filenames:
342 # Ignore files like .zuul.yaml.
343 if filename.startswith("."):
344 continue
303 if filename.endswith(".yaml"): 345 if filename.endswith(".yaml"):
304 yield os.path.join(root, filename) 346 yield os.path.join(root, filename)
305 347
diff --git a/pegleg/engine/util/pegleg_secret_management.py b/pegleg/engine/util/pegleg_secret_management.py
index 0937e2a..993e2d3 100644
--- a/pegleg/engine/util/pegleg_secret_management.py
+++ b/pegleg/engine/util/pegleg_secret_management.py
@@ -44,8 +44,7 @@ class PeglegSecretManagement():
44 44
45 if all([file_path, docs]) or \ 45 if all([file_path, docs]) or \
46 not any([file_path, docs]): 46 not any([file_path, docs]):
47 raise ValueError( 47 raise ValueError('Either `file_path` or `docs` must be specified.')
48 'Either `file_path` or `docs` must be specified.')
49 48
50 self.__check_environment() 49 self.__check_environment()
51 self.file_path = file_path 50 self.file_path = file_path
@@ -73,7 +72,7 @@ class PeglegSecretManagement():
73 # Verify that passphrase environment variable is defined and is longer 72 # Verify that passphrase environment variable is defined and is longer
74 # than 24 characters. 73 # than 24 characters.
75 if not os.environ.get(ENV_PASSPHRASE) or not re.match( 74 if not os.environ.get(ENV_PASSPHRASE) or not re.match(
76 PASSPHRASE_PATTERN, os.environ.get(ENV_PASSPHRASE)): 75 PASSPHRASE_PATTERN, os.environ.get(ENV_PASSPHRASE)):
77 raise click.ClickException( 76 raise click.ClickException(
78 'Environment variable {} is not defined or ' 77 'Environment variable {} is not defined or '
79 'is not at least 24-character long.'.format(ENV_PASSPHRASE)) 78 'is not at least 24-character long.'.format(ENV_PASSPHRASE))
@@ -154,8 +153,7 @@ class PeglegSecretManagement():
154 # do not decrypt already decrypted data 153 # do not decrypt already decrypted data
155 if doc.is_encrypted(): 154 if doc.is_encrypted():
156 doc.set_secret( 155 doc.set_secret(
157 decrypt(doc.get_secret(), 156 decrypt(doc.get_secret(), self.passphrase,
158 self.passphrase,
159 self.salt).decode()) 157 self.salt).decode())
160 doc.set_decrypted() 158 doc.set_decrypted()
161 doc_list.append(doc.embedded_document) 159 doc_list.append(doc.embedded_document)
diff --git a/tests/unit/engine/test_encryption.py b/tests/unit/engine/test_encryption.py
index a5869e8..33e00f8 100644
--- a/tests/unit/engine/test_encryption.py
+++ b/tests/unit/engine/test_encryption.py
@@ -30,7 +30,6 @@ from pegleg.engine.util.pegleg_secret_management import ENV_SALT
30from tests.unit.fixtures import temp_path 30from tests.unit.fixtures import temp_path
31from pegleg.engine.util import files 31from pegleg.engine.util import files
32 32
33
34TEST_DATA = """ 33TEST_DATA = """
35--- 34---
36schema: deckhand/Passphrase/v1 35schema: deckhand/Passphrase/v1
@@ -60,22 +59,24 @@ def test_encrypt_and_decrypt():
60 59
61 60
62@mock.patch.dict(os.environ, { 61@mock.patch.dict(os.environ, {
63 ENV_PASSPHRASE:'aShortPassphrase', 62 ENV_PASSPHRASE: 'aShortPassphrase',
64 ENV_SALT: 'MySecretSalt'}) 63 ENV_SALT: 'MySecretSalt'
64})
65def test_short_passphrase(): 65def test_short_passphrase():
66 with pytest.raises(click.ClickException, 66 with pytest.raises(
67 match=r'.*is not at least 24-character long.*'): 67 click.ClickException,
68 match=r'.*is not at least 24-character long.*'):
68 PeglegSecretManagement('file_path') 69 PeglegSecretManagement('file_path')
69 70
70 71
71def test_PeglegManagedDocument(): 72def test_pegleg_secret_management_constructor():
72 test_data = yaml.load(TEST_DATA) 73 test_data = yaml.load(TEST_DATA)
73 doc = PeglegManagedSecretsDocument(test_data) 74 doc = PeglegManagedSecretsDocument(test_data)
74 assert doc.is_storage_policy_encrypted() is True 75 assert doc.is_storage_policy_encrypted()
75 assert doc.is_encrypted() is False 76 assert not doc.is_encrypted()
76 77
77 78
78def test_PeglegSecretManagement(): 79def test_pegleg_secret_management_constructor_with_invalid_arguments():
79 with pytest.raises(ValueError) as err_info: 80 with pytest.raises(ValueError) as err_info:
80 PeglegSecretManagement(file_path=None, docs=None) 81 PeglegSecretManagement(file_path=None, docs=None)
81 assert 'Either `file_path` or `docs` must be specified.' in str( 82 assert 'Either `file_path` or `docs` must be specified.' in str(
@@ -87,40 +88,24 @@ def test_PeglegSecretManagement():
87 88
88 89
89@mock.patch.dict(os.environ, { 90@mock.patch.dict(os.environ, {
90 ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC', 91 ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
91 ENV_SALT: 'MySecretSalt'}) 92 ENV_SALT: 'MySecretSalt'
92def test_encrypt_file(): 93})
93 # write the test data to temp file 94def test_encrypt_decrypt_using_file_path(temp_path):
94 test_data = yaml.load(TEST_DATA)
95 dir = tempfile.mkdtemp()
96 file_path = os.path.join(dir, 'secrets_file.yaml')
97 save_path = os.path.join(dir, 'encrypted_secrets_file.yaml')
98 with open(file_path, 'w') as stream:
99 yaml.dump(test_data,
100 stream,
101 explicit_start=True,
102 explicit_end=True,
103 default_flow_style=False)
104 # read back the secrets data file and encrypt it
105 doc_mgr = PeglegSecretManagement(file_path)
106 doc_mgr.encrypt_secrets(save_path, 'test_author')
107 doc = doc_mgr.documents[0]
108 assert doc.is_encrypted()
109 assert doc.data['encrypted']['by'] == 'test_author'
110
111
112@mock.patch.dict(os.environ, {
113 ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC',
114 ENV_SALT: 'MySecretSalt'})
115def test_encrypt_decrypt_file(temp_path):
116 # write the test data to temp file 95 # write the test data to temp file
117 test_data = list(yaml.safe_load_all(TEST_DATA)) 96 test_data = list(yaml.safe_load_all(TEST_DATA))
118 file_path = os.path.join(temp_path, 'secrets_file.yaml') 97 file_path = os.path.join(temp_path, 'secrets_file.yaml')
119 files.write(file_path, test_data) 98 files.write(file_path, test_data)
120 save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml') 99 save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml')
100
101 # encrypt documents and validate that they were encrypted
121 doc_mgr = PeglegSecretManagement(file_path=file_path) 102 doc_mgr = PeglegSecretManagement(file_path=file_path)
122 doc_mgr.encrypt_secrets(save_path, 'test_author') 103 doc_mgr.encrypt_secrets(save_path, 'test_author')
123 # read back the encrypted file 104 doc = doc_mgr.documents[0]
105 assert doc.is_encrypted()
106 assert doc.data['encrypted']['by'] == 'test_author'
107
108 # decrypt documents and validate that they were decrypted
124 doc_mgr = PeglegSecretManagement(save_path) 109 doc_mgr = PeglegSecretManagement(save_path)
125 decrypted_data = doc_mgr.get_decrypted_secrets() 110 decrypted_data = doc_mgr.get_decrypted_secrets()
126 assert test_data[0]['data'] == decrypted_data[0]['data'] 111 assert test_data[0]['data'] == decrypted_data[0]['data']
@@ -128,23 +113,31 @@ def test_encrypt_decrypt_file(temp_path):
128 113
129 114
130@mock.patch.dict(os.environ, { 115@mock.patch.dict(os.environ, {
131 ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC', 116 ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
132 ENV_SALT: 'MySecretSalt'}) 117 ENV_SALT: 'MySecretSalt'
133def test_decrypt_document(temp_path): 118})
119def test_encrypt_decrypt_using_docs(temp_path):
134 # write the test data to temp file 120 # write the test data to temp file
135 test_data = list(yaml.safe_load_all(TEST_DATA)) 121 test_data = list(yaml.safe_load_all(TEST_DATA))
136 save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml') 122 save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml')
123
124 # encrypt documents and validate that they were encrypted
137 doc_mgr = PeglegSecretManagement(docs=test_data) 125 doc_mgr = PeglegSecretManagement(docs=test_data)
138 doc_mgr.encrypt_secrets(save_path, 'test_author') 126 doc_mgr.encrypt_secrets(save_path, 'test_author')
127 doc = doc_mgr.documents[0]
128 assert doc.is_encrypted()
129 assert doc.data['encrypted']['by'] == 'test_author'
130
139 # read back the encrypted file 131 # read back the encrypted file
140 with open(save_path) as stream: 132 with open(save_path) as stream:
141 encrypted_data = list(yaml.safe_load_all(stream)) 133 encrypted_data = list(yaml.safe_load_all(stream))
142 # this time pass a list of dicts to peglegSecretManager 134
135 # decrypt documents and validate that they were decrypted
143 doc_mgr = PeglegSecretManagement(docs=encrypted_data) 136 doc_mgr = PeglegSecretManagement(docs=encrypted_data)
144 decrypted_data = doc_mgr.get_decrypted_secrets() 137 decrypted_data = doc_mgr.get_decrypted_secrets()
145 assert test_data[0]['data'] == decrypted_data[0]['data'] 138 assert test_data[0]['data'] == decrypted_data[0]['data']
146 assert test_data[0]['schema'] == decrypted_data[0]['schema'] 139 assert test_data[0]['schema'] == decrypted_data[0]['schema']
147 assert test_data[0]['metadata']['name'] == decrypted_data[0][ 140 assert test_data[0]['metadata']['name'] == decrypted_data[0]['metadata'][
148 'metadata']['name'] 141 'name']
149 assert test_data[0]['metadata']['storagePolicy'] == decrypted_data[0][ 142 assert test_data[0]['metadata']['storagePolicy'] == decrypted_data[0][
150 'metadata']['storagePolicy'] 143 'metadata']['storagePolicy']
diff --git a/tests/unit/engine/test_lint.py b/tests/unit/engine/test_lint.py
index e15748e..ce096c4 100644
--- a/tests/unit/engine/test_lint.py
+++ b/tests/unit/engine/test_lint.py
@@ -125,23 +125,6 @@ def test_verify_deckhand_render_site_documents_separately(
125 'storagePolicy': 'cleartext' 125 'storagePolicy': 'cleartext'
126 }, 126 },
127 'schema': 'deckhand/Passphrase/v1' 127 'schema': 'deckhand/Passphrase/v1'
128 }, {
129 'data': {
130 'site_type': sitename,
131 'repositories': {
132 'global': mock.ANY
133 }
134 },
135 'metadata': {
136 'layeringDefinition': {
137 'abstract': False,
138 'layer': 'site'
139 },
140 'name': sitename,
141 'schema': 'metadata/Document/v1',
142 'storagePolicy': 'cleartext'
143 },
144 'schema': 'pegleg/SiteDefinition/v1'
145 }] 128 }]
146 expected_documents.extend(documents) 129 expected_documents.extend(documents)
147 130
diff --git a/tests/unit/engine/util/test_files.py b/tests/unit/engine/util/test_files.py
new file mode 100644
index 0000000..b0938ee
--- /dev/null
+++ b/tests/unit/engine/util/test_files.py
@@ -0,0 +1,38 @@
1# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import os
16
17from pegleg import config
18from pegleg.engine.util import files
19from tests.unit.fixtures import create_tmp_deployment_files
20
21
22class TestFileHelpers(object):
23 def test_read_compatible_file(self, create_tmp_deployment_files):
24 path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'secrets',
25 'passphrases', 'cicd-passphrase.yaml')
26 documents = files.read(path)
27 assert 1 == len(documents)
28
29 def test_read_incompatible_file(self, create_tmp_deployment_files):
30 # NOTE(felipemonteiro): The Pegleg site-definition.yaml is a
31 # Deckhand-formatted document currently but probably shouldn't be,
32 # because it has no business being in Deckhand. As such, validate that
33 # it is ignored.
34 path = os.path.join(config.get_site_repo(), 'site', 'cicd',
35 'site-definition.yaml')
36 documents = files.read(path)
37 assert not documents, ("Documents returned should be empty for "
38 "site-definition.yaml")