diff --git a/doc/source/cli/cli.rst b/doc/source/cli/cli.rst index 94b89dae..b93d6c10 100644 --- a/doc/source/cli/cli.rst +++ b/doc/source/cli/cli.rst @@ -612,6 +612,42 @@ Example: secrets decrypt site1 -f \ /opt/security-manifests/site/site1/passwords/password1.yaml +genesis_bundle +-------------- + +Constructs genesis bundle based on a site configuration. + +.. note:: + This command requires the environment variable PEGLEG_PASSPHRASE + to be set and at least 24 characters long, to be used for encrypting + genesis bundle data. PEGLEG_SALT must be set as well. There are no + constraints on its length, but at least 24 characters is recommended. + + +**-b / --build-dir** (Required). + +Destination directory for the genesis bundle. + +**--include-validators** (Optional). False by default. + +A flag to request build genesis validation scripts as well. + +Usage: + +:: + ./pegleg.sh site genesis_bundle \ + -b -k --validators + +Examples +^^^^^^^^ + +:: + + ./pegleg.sh site -r ./site-manifests \ + genesis_bundle site1 \ + -b ../../site1_build \ + -k yourEncryptionPassphrase \ + --validators generate ^^^^^^^^ @@ -803,8 +839,9 @@ Where mandatory encrypted schema type is one of: P002 - Deckhand rendering is expected to complete without errors. P003 - All repos contain expected directories. -.. _Deckhand: https://airship-deckhand.readthedocs.io/en/latest/rendering.html -.. _Deckhand Validations: https://airship-deckhand.readthedocs.io/en/latest/validation.html + +.. _Deckhand: https://airship-deckhand.readthedocs.io/en/latest/users/rendering.html +.. _Deckhand Validations: https://airship-deckhand.readthedocs.io/en/latest/overview.html#validation .. _Pegleg Managed Documents: https://airship-specs.readthedocs.io/en/latest/specs/approved/pegleg-secrets.html#peglegmanageddocument .. _Shipyard: https://github.com/openstack/airship-shipyard .. _CLI documentation: https://airship-shipyard.readthedocs.io/en/latest/CLI.html#openstack-keystone-authorization-environment-variables @@ -878,4 +915,4 @@ Example with length specified: :: - ./pegleg.sh generate salt -l \ No newline at end of file + ./pegleg.sh generate salt -l diff --git a/doc/source/exceptions.rst b/doc/source/exceptions.rst index 06585f89..24ec61b2 100644 --- a/doc/source/exceptions.rst +++ b/doc/source/exceptions.rst @@ -68,10 +68,19 @@ PKI Exceptions -------------- .. autoexception:: pegleg.engine.exceptions.IncompletePKIPairError + +Genesis Bundle Exceptions +------------------------- + +.. autoexception:: pegleg.engine.exceptions.GenesisBundleEncryptionException :members: :show-inheritance: :undoc-members: +.. autoexception:: pegleg.engine.exceptions.GenesisBundleGenerateException + :members: + :show-inheritance: + Passphrase Exceptions --------------------- diff --git a/pegleg/cli.py b/pegleg/cli.py index 0d93223f..52a6f8b7 100644 --- a/pegleg/cli.py +++ b/pegleg/cli.py @@ -14,13 +14,16 @@ import functools import logging +import os import sys import click from pegleg import config from pegleg import engine +from pegleg.engine import bundle from pegleg.engine import catalog +from pegleg.engine.util.pegleg_secret_management import PeglegSecretManagement from pegleg.engine.util.shipyard_helper import ShipyardHelper LOG = logging.getLogger(__name__) @@ -412,6 +415,54 @@ def generate_pki(site_name, author): click.echo("Generated PKI files written to:\n%s" % '\n'.join(output_paths)) +@site.command( + 'genesis_bundle', + help='Construct the genesis deployment bundle.') +@click.option( + '-b', + '--build-dir', + 'build_dir', + type=click.Path(file_okay=False, dir_okay=True, resolve_path=True), + required=True, + help='Destination directory to store the genesis bundle.') +@click.option( + '--include-validators', + 'validators', + is_flag=True, + default=False, + help='A flag to request generate genesis validation scripts in addition ' + 'to genesis.sh script.') +@SITE_REPOSITORY_ARGUMENT +def genesis_bundle(*, build_dir, validators, site_name): + prom_encryption_key = os.environ.get("PROMENADE_ENCRYPTION_KEY") + peg_encryption_key = os.environ.get("PEGLEG_PASSPHRASE") + encryption_key = None + if (prom_encryption_key and len(prom_encryption_key) > 24 and + peg_encryption_key and len(peg_encryption_key) > 24): + click.echo("WARNING: PROMENADE_ENCRYPTION_KEY is deprecated, " + "using PEGLEG_PASSPHRASE instead", err=True) + config.set_passphrase(peg_encryption_key) + encryption_key = peg_encryption_key + elif prom_encryption_key and len(prom_encryption_key) > 24: + click.echo("ERROR: PROMENADE_ENCRYPTION_KEY is deprecated, " + "use PEGLEG_PASSPHRASE instead", err=True) + raise click.ClickException("ERROR: PEGLEG_PASSPHRASE must be set " + "and at least 24 characters long.") + elif peg_encryption_key and len(peg_encryption_key) > 24: + config.set_passphrase(peg_encryption_key) + encryption_key = peg_encryption_key + else: + raise click.ClickException("ERROR: PEGLEG_PASSPHRASE must be set " + "and at least 24 characters long.") + + PeglegSecretManagement.check_environment() + bundle.build_genesis(build_dir, + encryption_key, + validators, + logging.DEBUG == LOG.getEffectiveLevel(), + site_name) + + @main.group(help='Commands related to types') @MAIN_REPOSITORY_OPTION @REPOSITORY_CLONE_PATH_OPTION diff --git a/pegleg/config.py b/pegleg/config.py index be007cae..a481e4f2 100644 --- a/pegleg/config.py +++ b/pegleg/config.py @@ -26,7 +26,9 @@ except NameError: 'clone_path': None, 'site_path': 'site', 'site_rev': None, - 'type_path': 'type' + 'type_path': 'type', + 'passphrase': None, + 'salt': None } @@ -147,3 +149,23 @@ def set_rel_type_path(p): """Set the relative type path name.""" p = p or 'type' GLOBAL_CONTEXT['type_path'] = p + + +def set_passphrase(p): + """Set the passphrase for encryption and decryption.""" + GLOBAL_CONTEXT['passphrase'] = p + + +def get_passphrase(): + """Get the passphrase for encryption and decryption.""" + return GLOBAL_CONTEXT['passphrase'] + + +def set_salt(p): + """Set the salt for encryption and decryption.""" + GLOBAL_CONTEXT['salt'] = p + + +def get_salt(): + """Get the salt for encryption and decryption.""" + return GLOBAL_CONTEXT['salt'] diff --git a/pegleg/engine/bundle.py b/pegleg/engine/bundle.py new file mode 100644 index 00000000..c498298e --- /dev/null +++ b/pegleg/engine/bundle.py @@ -0,0 +1,92 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import stat + +import click + +from pegleg.engine.exceptions import GenesisBundleEncryptionException +from pegleg.engine.exceptions import GenesisBundleGenerateException +from pegleg.engine import util +from pegleg.engine.util.pegleg_secret_management import PeglegSecretManagement + +from promenade.builder import Builder +from promenade.config import Configuration +from promenade import exceptions + +LOG = logging.getLogger(__name__) + +__all__ = [ + 'build_genesis', +] + + +def build_genesis(build_path, encryption_key, validators, debug, site_name): + """ + Build the genesis deployment bundle, and store it in ``build_path``. + + Build the genesis.sh script, base65-encode, encrypt and embed the + site configuration source documents in genesis.sh script. + If ``validators`` flag should be True, build the bundle validator + scripts as well. + Store the built deployment bundle in `build_path`. + + :param str build_path: Directory path of the built genesis deployment + bundle + :param str encryption_key: Key to use to encrypt the bundled site + configuration in genesis.sh script. + :param bool validators: Whether to generate validator scripts + :param int debug: pegleg debug level to pass to promenade engine + for logging. + :return: None + """ + + # Raise an error if the build path exists. We don't want to overwrite it. + if os.path.isdir(build_path): + raise click.ClickException( + "{} already exists, remove it or specify a new " + "directory.".format(build_path)) + # Get the list of config files + LOG.info('=== Building bootstrap scripts ===') + + # Copy the site config, and site secrets to build directory + os.mkdir(build_path) + os.chmod(build_path, os.stat(build_path).st_mode | stat.S_IRWXU | + stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH) + documents = util.definition.documents_for_site(site_name) + secret_manager = PeglegSecretManagement(docs=documents) + documents = secret_manager.get_decrypted_secrets() + try: + # Use the promenade engine to build and encrypt the genesis bundle + c = Configuration( + documents=documents, + debug=debug, + substitute=True, + allow_missing_substitutions=False, + leave_kubectl=False) + if c.get_path('EncryptionPolicy:scripts.genesis') and encryption_key: + os.environ['PROMENADE_ENCRYPTION_KEY'] = encryption_key + os.environ['PEGLEG_PASSPHRASE'] = encryption_key + Builder(c, validators=validators).build_all(output_dir=build_path) + else: + raise GenesisBundleEncryptionException() + + except exceptions.PromenadeException as e: + LOG.error('Build genesis bundle failed! {}.'.format( + e.display(debug=debug))) + raise GenesisBundleGenerateException() + + LOG.info('=== Done! ===') diff --git a/pegleg/engine/exceptions.py b/pegleg/engine/exceptions.py index e26c7186..d96f519d 100644 --- a/pegleg/engine/exceptions.py +++ b/pegleg/engine/exceptions.py @@ -86,3 +86,19 @@ class PassphraseCatalogNotFoundException(PeglegBaseException): """Failed to find Catalog for Passphrases generation.""" message = ('Could not find the Passphrase Catalog to generate ' 'the site Passphrases!') + + +class GenesisBundleEncryptionException(PeglegBaseException): + """Exception raised when encryption of the genesis bundle fails.""" + + message = 'Encryption is required for genesis bundle, but no encryption ' \ + 'policy or key is specified.' + + +class GenesisBundleGenerateException(PeglegBaseException): + """ + Exception raised when pormenade engine fails to build the genesis + bundle. + """ + + message = 'Bundle generation failed on deckhand validation.' diff --git a/pegleg/engine/util/pegleg_secret_management.py b/pegleg/engine/util/pegleg_secret_management.py index 78e97ed0..7a18589d 100644 --- a/pegleg/engine/util/pegleg_secret_management.py +++ b/pegleg/engine/util/pegleg_secret_management.py @@ -19,6 +19,7 @@ import re import click import yaml +from pegleg import config from pegleg.engine.util.encryption import decrypt from pegleg.engine.util.encryption import encrypt from pegleg.engine.util import files @@ -47,10 +48,11 @@ class PeglegSecretManagement(object): raise ValueError('Either `file_path` or `docs` must be ' 'specified.') - if generated and not (author and catalog): + if generated and not (catalog and author): raise ValueError("If the document is generated, author and " "catalog must be specified.") - self.__check_environment() + + self.check_environment() self.file_path = file_path self.documents = list() self._generated = generated @@ -68,8 +70,17 @@ class PeglegSecretManagement(object): self._author = author - self.passphrase = os.environ.get(ENV_PASSPHRASE).encode() - self.salt = os.environ.get(ENV_SALT).encode() + if config.get_passphrase() and config.get_salt(): + self.passphrase = config.get_passphrase() + self.salt = config.get_salt() + elif config.get_passphrase() or config.get_salt(): + raise ValueError("ERROR: Pegleg configuration must either have " + "both a passphrase and a salt or neither.") + else: + self.passphrase = os.environ.get(ENV_PASSPHRASE).encode() + self.salt = os.environ.get(ENV_SALT).encode() + config.set_passphrase(self.passphrase) + config.set_salt(self.salt) def __iter__(self): """ @@ -79,7 +90,7 @@ class PeglegSecretManagement(object): return (doc.pegleg_document for doc in self.documents) @staticmethod - def __check_environment(): + def check_environment(): """ Validate required environment variables for encryption or decryption. diff --git a/requirements.txt b/requirements.txt index 41d4561b..4f97c4c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ python-dateutil==2.7.3 rstr==2.2.6 git+https://github.com/openstack/airship-deckhand.git@49ad9f38842f7f1ecb86d907d86d332f8186eb8c git+https://github.com/openstack/airship-shipyard.git@44f7022df6438de541501c2fdd5c46df198b82bf#egg=shipyard_client&subdirectory=src/bin/shipyard_client +git+https://github.com/openstack/airship-promenade.git@a6e8fdbe22bd153c78a008b92cd5d1c245bc63e3 diff --git a/tests/unit/engine/test_build_genesis_bundle.py b/tests/unit/engine/test_build_genesis_bundle.py new file mode 100644 index 00000000..21baba93 --- /dev/null +++ b/tests/unit/engine/test_build_genesis_bundle.py @@ -0,0 +1,144 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import mock +import pytest +import yaml + +from pegleg import config +from pegleg.engine import bundle +from pegleg.engine.exceptions import GenesisBundleEncryptionException +from pegleg.engine.exceptions import GenesisBundleGenerateException +from pegleg.engine.util import files +from pegleg.engine.util.pegleg_secret_management import ENV_PASSPHRASE +from pegleg.engine.util.pegleg_secret_management import ENV_SALT + +from tests.unit.fixtures import temp_path + +SITE_DEFINITION = """ +--- +# High-level pegleg site definition file +schema: pegleg/SiteDefinition/v1 +metadata: + schema: metadata/Document/v1 + layeringDefinition: + abstract: false + layer: site + # NEWSITE-CHANGEME: Replace with the site name + name: test_site + storagePolicy: cleartext +data: + # The type layer this site will delpoy with. Type layer is found in the + # type folder. + site_type: foundry +... + +""" + +SITE_CONFIG_DATA = """ +--- +schema: promenade/EncryptionPolicy/v1 +metadata: + schema: metadata/Document/v1 + name: encryption-policy + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + scripts: + genesis: + gpg: {} + join: + gpg: {} +--- +schema: deckhand/LayeringPolicy/v1 +metadata: + schema: metadata/Control/v1 + name: layering-policy +data: + layerOrder: + - global + - type + - site +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: ceph_swift_keystone_password + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: ABAgagajajkb839215387 +... +""" + + +@mock.patch.dict(os.environ, { + ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC', + ENV_SALT: 'MySecretSalt' +}) +def test_no_encryption_key(temp_path): + # Write the test data to temp file + config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA)) + base_config_dir = os.path.join(temp_path, 'config_dir') + config.set_site_repo(base_config_dir) + config_dir = os.path.join(base_config_dir, 'site', 'test_site') + + config_path = os.path.join(config_dir, 'config_file.yaml') + build_dir = os.path.join(temp_path, 'build_dir') + os.makedirs(config_dir) + + files.write(config_path, config_data) + files.write(os.path.join(config_dir, "site-definition.yaml"), + yaml.safe_load_all(SITE_DEFINITION)) + + with pytest.raises(GenesisBundleEncryptionException, + match=r'.*no encryption policy or key is specified.*'): + bundle.build_genesis(build_path=build_dir, + encryption_key=None, + validators=False, + debug=logging.ERROR, + site_name="test_site") + + +@mock.patch.dict(os.environ, { + ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC', + ENV_SALT: 'MySecretSalt' +}) +def test_failed_deckhand_validation(temp_path): + # Write the test data to temp file + config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA)) + base_config_dir = os.path.join(temp_path, 'config_dir') + config.set_site_repo(base_config_dir) + config_dir = os.path.join(base_config_dir, 'site', 'test_site') + + config_path = os.path.join(config_dir, 'config_file.yaml') + build_dir = os.path.join(temp_path, 'build_dir') + os.makedirs(config_dir) + files.write(config_path, config_data) + files.write(os.path.join(config_dir, "site-definition.yaml"), + yaml.safe_load_all(SITE_DEFINITION)) + key = 'MyverYSecretEncryptionKey382803' + with pytest.raises(GenesisBundleGenerateException, + match=r'.*failed on deckhand validation.*'): + bundle.build_genesis(build_path=build_dir, + encryption_key=key, + validators=False, + debug=logging.ERROR, + site_name="test_site")