diff --git a/pegleg/cli.py b/pegleg/cli.py index 0e3614df..a7c3b43c 100644 --- a/pegleg/cli.py +++ b/pegleg/cli.py @@ -385,7 +385,7 @@ def secrets(): 'author', required=True, help='Identifier for the program or person who is encrypting the secrets ' - 'documents') + 'documents') @click.argument('site_name') def encrypt(*, save_location, author, site_name): engine.repository.process_repositories(site_name) diff --git a/pegleg/engine/lint.py b/pegleg/engine/lint.py index 550c0654..c0706ce4 100644 --- a/pegleg/engine/lint.py +++ b/pegleg/engine/lint.py @@ -18,7 +18,6 @@ import os import pkg_resources import shutil import textwrap -import yaml from prettytable import PrettyTable @@ -223,16 +222,16 @@ def _verify_single_file(filename, schemas): errors.append((FILE_MISSING_YAML_DOCUMENT_HEADER, '%s does not begin with YAML beginning of document ' 'marker "---".' % filename)) - f.seek(0) - documents = [] - try: - documents = list(yaml.safe_load_all(f)) - except Exception as e: - errors.append((FILE_CONTAINS_INVALID_YAML, - '%s is not valid yaml: %s' % (filename, e))) - for document in documents: - errors.extend(_verify_document(document, schemas, filename)) + documents = [] + try: + documents = util.files.read(filename) + except Exception as e: + errors.append((FILE_CONTAINS_INVALID_YAML, + '%s is not valid yaml: %s' % (filename, e))) + + for document in documents: + errors.extend(_verify_document(document, schemas, filename)) return errors diff --git a/pegleg/engine/util/definition.py b/pegleg/engine/util/definition.py index 117d2a31..1b2c39c9 100644 --- a/pegleg/engine/util/definition.py +++ b/pegleg/engine/util/definition.py @@ -15,7 +15,6 @@ import os import click -import yaml from pegleg import config from pegleg.engine.util import files @@ -52,6 +51,7 @@ def load_as_params(site_name, *fields, primary_repo_base=None): def path(site_name, primary_repo_base=None): + """Retrieve path to the site-definition.yaml file for ``site_name``.""" if not primary_repo_base: primary_repo_base = config.get_site_repo() return os.path.join(primary_repo_base, 'site', site_name, @@ -100,8 +100,7 @@ def documents_for_each_site(): paths = files.directories_for(**params) filenames = set(files.search(paths)) for filename in filenames: - with open(filename) as f: - documents[sitename].extend(list(yaml.safe_load_all(f))) + documents[sitename].extend(files.read(filename)) return documents @@ -122,7 +121,6 @@ def documents_for_site(sitename): paths = files.directories_for(**params) filenames = set(files.search(paths)) for filename in filenames: - with open(filename) as f: - documents.extend(list(yaml.safe_load_all(f))) + documents.extend(files.read(filename)) return documents diff --git a/pegleg/engine/util/files.py b/pegleg/engine/util/files.py index 5281dbb4..0ab0cae9 100644 --- a/pegleg/engine/util/files.py +++ b/pegleg/engine/util/files.py @@ -18,6 +18,7 @@ import yaml import logging from pegleg import config +from pegleg.engine.util import pegleg_managed_document as md LOG = logging.getLogger(__name__) @@ -248,9 +249,35 @@ def read(path): '{} not found. Pegleg must be run from the root of a ' 'configuration repository.'.format(path)) + def is_deckhand_document(document): + # Deckhand documents only consist of control and application + # documents. + valid_schemas = ('metadata/Control', 'metadata/Document') + if isinstance(document, dict): + schema = document.get('metadata', {}).get('schema', '') + # NOTE(felipemonteiro): The Pegleg site-definition.yaml is a + # Deckhand-formatted document currently but probably shouldn't + # be, because it has no business being in Deckhand. As such, + # treat it as a special case. + if "SiteDefinition" in document.get('schema', ''): + return False + if any(schema.startswith(x) for x in valid_schemas): + return True + else: + LOG.debug('Document with schema=%s is not a valid Deckhand ' + 'schema. Ignoring it.', schema) + return False + + def is_pegleg_managed_document(document): + return md.PeglegManagedSecretsDocument.is_pegleg_managed_secret( + document) + with open(path) as stream: try: - return list(yaml.safe_load_all(stream)) + return [ + d for d in yaml.safe_load_all(stream) + if is_deckhand_document(d) or is_pegleg_managed_document(d) + ] except yaml.YAMLError as e: raise click.ClickException('Failed to parse %s:\n%s' % (path, e)) @@ -296,10 +323,25 @@ def _recurse_subdirs(search_path, depth): def search(search_paths): + if not isinstance(search_paths, (list, tuple)): + search_paths = [search_paths] + for search_path in search_paths: LOG.debug("Recursively collecting YAMLs from %s" % search_path) - for root, _dirs, filenames in os.walk(search_path): + for root, _, filenames in os.walk(search_path): + + # Ignore hidden folders like .tox or .git for faster processing. + if os.path.basename(root).startswith("."): + continue + # Skip over anything in tools/ because it will never contain valid + # Pegleg-owned manifest documents. + if "tools" in root.split("/"): + continue + for filename in filenames: + # Ignore files like .zuul.yaml. + if filename.startswith("."): + continue if filename.endswith(".yaml"): yield os.path.join(root, filename) diff --git a/pegleg/engine/util/pegleg_secret_management.py b/pegleg/engine/util/pegleg_secret_management.py index 0937e2ac..993e2d31 100644 --- a/pegleg/engine/util/pegleg_secret_management.py +++ b/pegleg/engine/util/pegleg_secret_management.py @@ -44,8 +44,7 @@ class PeglegSecretManagement(): if all([file_path, docs]) or \ not any([file_path, docs]): - raise ValueError( - 'Either `file_path` or `docs` must be specified.') + raise ValueError('Either `file_path` or `docs` must be specified.') self.__check_environment() self.file_path = file_path @@ -73,7 +72,7 @@ class PeglegSecretManagement(): # Verify that passphrase environment variable is defined and is longer # than 24 characters. if not os.environ.get(ENV_PASSPHRASE) or not re.match( - PASSPHRASE_PATTERN, os.environ.get(ENV_PASSPHRASE)): + PASSPHRASE_PATTERN, os.environ.get(ENV_PASSPHRASE)): raise click.ClickException( 'Environment variable {} is not defined or ' 'is not at least 24-character long.'.format(ENV_PASSPHRASE)) @@ -154,8 +153,7 @@ class PeglegSecretManagement(): # do not decrypt already decrypted data if doc.is_encrypted(): doc.set_secret( - decrypt(doc.get_secret(), - self.passphrase, + decrypt(doc.get_secret(), self.passphrase, self.salt).decode()) doc.set_decrypted() doc_list.append(doc.embedded_document) diff --git a/tests/unit/engine/test_encryption.py b/tests/unit/engine/test_encryption.py index a5869e87..33e00f85 100644 --- a/tests/unit/engine/test_encryption.py +++ b/tests/unit/engine/test_encryption.py @@ -30,7 +30,6 @@ from pegleg.engine.util.pegleg_secret_management import ENV_SALT from tests.unit.fixtures import temp_path from pegleg.engine.util import files - TEST_DATA = """ --- schema: deckhand/Passphrase/v1 @@ -60,22 +59,24 @@ def test_encrypt_and_decrypt(): @mock.patch.dict(os.environ, { - ENV_PASSPHRASE:'aShortPassphrase', - ENV_SALT: 'MySecretSalt'}) + ENV_PASSPHRASE: 'aShortPassphrase', + ENV_SALT: 'MySecretSalt' +}) def test_short_passphrase(): - with pytest.raises(click.ClickException, - match=r'.*is not at least 24-character long.*'): + with pytest.raises( + click.ClickException, + match=r'.*is not at least 24-character long.*'): PeglegSecretManagement('file_path') -def test_PeglegManagedDocument(): +def test_pegleg_secret_management_constructor(): test_data = yaml.load(TEST_DATA) doc = PeglegManagedSecretsDocument(test_data) - assert doc.is_storage_policy_encrypted() is True - assert doc.is_encrypted() is False + assert doc.is_storage_policy_encrypted() + assert not doc.is_encrypted() -def test_PeglegSecretManagement(): +def test_pegleg_secret_management_constructor_with_invalid_arguments(): with pytest.raises(ValueError) as err_info: PeglegSecretManagement(file_path=None, docs=None) assert 'Either `file_path` or `docs` must be specified.' in str( @@ -87,40 +88,24 @@ def test_PeglegSecretManagement(): @mock.patch.dict(os.environ, { - ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC', - ENV_SALT: 'MySecretSalt'}) -def test_encrypt_file(): - # write the test data to temp file - test_data = yaml.load(TEST_DATA) - dir = tempfile.mkdtemp() - file_path = os.path.join(dir, 'secrets_file.yaml') - save_path = os.path.join(dir, 'encrypted_secrets_file.yaml') - with open(file_path, 'w') as stream: - yaml.dump(test_data, - stream, - explicit_start=True, - explicit_end=True, - default_flow_style=False) - # read back the secrets data file and encrypt it - doc_mgr = PeglegSecretManagement(file_path) - doc_mgr.encrypt_secrets(save_path, 'test_author') - doc = doc_mgr.documents[0] - assert doc.is_encrypted() - assert doc.data['encrypted']['by'] == 'test_author' - - -@mock.patch.dict(os.environ, { - ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC', - ENV_SALT: 'MySecretSalt'}) -def test_encrypt_decrypt_file(temp_path): + ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC', + ENV_SALT: 'MySecretSalt' +}) +def test_encrypt_decrypt_using_file_path(temp_path): # write the test data to temp file test_data = list(yaml.safe_load_all(TEST_DATA)) file_path = os.path.join(temp_path, 'secrets_file.yaml') files.write(file_path, test_data) save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml') + + # encrypt documents and validate that they were encrypted doc_mgr = PeglegSecretManagement(file_path=file_path) doc_mgr.encrypt_secrets(save_path, 'test_author') - # read back the encrypted file + doc = doc_mgr.documents[0] + assert doc.is_encrypted() + assert doc.data['encrypted']['by'] == 'test_author' + + # decrypt documents and validate that they were decrypted doc_mgr = PeglegSecretManagement(save_path) decrypted_data = doc_mgr.get_decrypted_secrets() assert test_data[0]['data'] == decrypted_data[0]['data'] @@ -128,23 +113,31 @@ def test_encrypt_decrypt_file(temp_path): @mock.patch.dict(os.environ, { - ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC', - ENV_SALT: 'MySecretSalt'}) -def test_decrypt_document(temp_path): + ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC', + ENV_SALT: 'MySecretSalt' +}) +def test_encrypt_decrypt_using_docs(temp_path): # write the test data to temp file test_data = list(yaml.safe_load_all(TEST_DATA)) save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml') + + # encrypt documents and validate that they were encrypted doc_mgr = PeglegSecretManagement(docs=test_data) doc_mgr.encrypt_secrets(save_path, 'test_author') + doc = doc_mgr.documents[0] + assert doc.is_encrypted() + assert doc.data['encrypted']['by'] == 'test_author' + # read back the encrypted file with open(save_path) as stream: encrypted_data = list(yaml.safe_load_all(stream)) - # this time pass a list of dicts to peglegSecretManager + + # decrypt documents and validate that they were decrypted doc_mgr = PeglegSecretManagement(docs=encrypted_data) decrypted_data = doc_mgr.get_decrypted_secrets() assert test_data[0]['data'] == decrypted_data[0]['data'] assert test_data[0]['schema'] == decrypted_data[0]['schema'] - assert test_data[0]['metadata']['name'] == decrypted_data[0][ - 'metadata']['name'] + assert test_data[0]['metadata']['name'] == decrypted_data[0]['metadata'][ + 'name'] assert test_data[0]['metadata']['storagePolicy'] == decrypted_data[0][ 'metadata']['storagePolicy'] diff --git a/tests/unit/engine/test_lint.py b/tests/unit/engine/test_lint.py index e15748e0..ce096c4c 100644 --- a/tests/unit/engine/test_lint.py +++ b/tests/unit/engine/test_lint.py @@ -125,23 +125,6 @@ def test_verify_deckhand_render_site_documents_separately( 'storagePolicy': 'cleartext' }, 'schema': 'deckhand/Passphrase/v1' - }, { - 'data': { - 'site_type': sitename, - 'repositories': { - 'global': mock.ANY - } - }, - 'metadata': { - 'layeringDefinition': { - 'abstract': False, - 'layer': 'site' - }, - 'name': sitename, - 'schema': 'metadata/Document/v1', - 'storagePolicy': 'cleartext' - }, - 'schema': 'pegleg/SiteDefinition/v1' }] expected_documents.extend(documents) diff --git a/tests/unit/engine/util/test_files.py b/tests/unit/engine/util/test_files.py new file mode 100644 index 00000000..b0938ee3 --- /dev/null +++ b/tests/unit/engine/util/test_files.py @@ -0,0 +1,38 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from pegleg import config +from pegleg.engine.util import files +from tests.unit.fixtures import create_tmp_deployment_files + + +class TestFileHelpers(object): + def test_read_compatible_file(self, create_tmp_deployment_files): + path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'secrets', + 'passphrases', 'cicd-passphrase.yaml') + documents = files.read(path) + assert 1 == len(documents) + + def test_read_incompatible_file(self, create_tmp_deployment_files): + # NOTE(felipemonteiro): The Pegleg site-definition.yaml is a + # Deckhand-formatted document currently but probably shouldn't be, + # because it has no business being in Deckhand. As such, validate that + # it is ignored. + path = os.path.join(config.get_site_repo(), 'site', 'cicd', + 'site-definition.yaml') + documents = files.read(path) + assert not documents, ("Documents returned should be empty for " + "site-definition.yaml")