summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAhmad Mahmoudi <am495p@att.com>2018-10-25 12:55:34 -0500
committerLev Morgan <lm734y@att.com>2019-03-07 03:00:30 -0600
commitc4f25b4d4fc47ea7c9fcd6cab63480fadbebf111 (patch)
tree334aaf18ea8c5268c41e46b03c7e7625423a84a4
parent50ce7a02e08a0a5277c2fbda96ece6eb5782407a (diff)
CLI: Add command to generate genesis bundle
Added a pegleg cli command to build genesis.sh bundle for a site deployment. Pegleg imports promenade engine, and uses promenade to build and encrypt the genesis.sh deployment bundle. Change-Id: I1a489459b2c56b7b53018c32aab5e6550c69e1d2
Notes
Notes (review): Code-Review+2: Scott Hussey <sthussey@att.com> Code-Review+2: Matt McEuen <matt.mceuen@att.com> Workflow+1: Matt McEuen <matt.mceuen@att.com> Code-Review+1: Dimitrios Markou <dimitrios.markou@att.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Mon, 11 Mar 2019 20:26:03 +0000 Reviewed-on: https://review.openstack.org/613392 Project: openstack/airship-pegleg Branch: refs/heads/master
-rw-r--r--doc/source/cli/cli.rst43
-rw-r--r--doc/source/exceptions.rst9
-rw-r--r--pegleg/cli.py51
-rw-r--r--pegleg/config.py24
-rw-r--r--pegleg/engine/bundle.py92
-rw-r--r--pegleg/engine/exceptions.py16
-rw-r--r--pegleg/engine/util/pegleg_secret_management.py21
-rw-r--r--requirements.txt1
-rw-r--r--tests/unit/engine/test_build_genesis_bundle.py144
9 files changed, 392 insertions, 9 deletions
diff --git a/doc/source/cli/cli.rst b/doc/source/cli/cli.rst
index 94b89da..b93d6c1 100644
--- a/doc/source/cli/cli.rst
+++ b/doc/source/cli/cli.rst
@@ -612,6 +612,42 @@ Example:
612 secrets decrypt site1 -f \ 612 secrets decrypt site1 -f \
613 /opt/security-manifests/site/site1/passwords/password1.yaml 613 /opt/security-manifests/site/site1/passwords/password1.yaml
614 614
615genesis_bundle
616--------------
617
618Constructs genesis bundle based on a site configuration.
619
620.. note::
621 This command requires the environment variable PEGLEG_PASSPHRASE
622 to be set and at least 24 characters long, to be used for encrypting
623 genesis bundle data. PEGLEG_SALT must be set as well. There are no
624 constraints on its length, but at least 24 characters is recommended.
625
626
627**-b / --build-dir** (Required).
628
629Destination directory for the genesis bundle.
630
631**--include-validators** (Optional). False by default.
632
633A flag to request build genesis validation scripts as well.
634
635Usage:
636
637::
638 ./pegleg.sh site <options> genesis_bundle <site_name> \
639 -b <build_locaton> -k <encryption_passphrase/key> --validators
640
641Examples
642^^^^^^^^
643
644::
645
646 ./pegleg.sh site -r ./site-manifests \
647 genesis_bundle site1 \
648 -b ../../site1_build \
649 -k yourEncryptionPassphrase \
650 --validators
615 651
616generate 652generate
617^^^^^^^^ 653^^^^^^^^
@@ -803,8 +839,9 @@ Where mandatory encrypted schema type is one of:
803P002 - Deckhand rendering is expected to complete without errors. 839P002 - Deckhand rendering is expected to complete without errors.
804P003 - All repos contain expected directories. 840P003 - All repos contain expected directories.
805 841
806.. _Deckhand: https://airship-deckhand.readthedocs.io/en/latest/rendering.html 842
807.. _Deckhand Validations: https://airship-deckhand.readthedocs.io/en/latest/validation.html 843.. _Deckhand: https://airship-deckhand.readthedocs.io/en/latest/users/rendering.html
844.. _Deckhand Validations: https://airship-deckhand.readthedocs.io/en/latest/overview.html#validation
808.. _Pegleg Managed Documents: https://airship-specs.readthedocs.io/en/latest/specs/approved/pegleg-secrets.html#peglegmanageddocument 845.. _Pegleg Managed Documents: https://airship-specs.readthedocs.io/en/latest/specs/approved/pegleg-secrets.html#peglegmanageddocument
809.. _Shipyard: https://github.com/openstack/airship-shipyard 846.. _Shipyard: https://github.com/openstack/airship-shipyard
810.. _CLI documentation: https://airship-shipyard.readthedocs.io/en/latest/CLI.html#openstack-keystone-authorization-environment-variables 847.. _CLI documentation: https://airship-shipyard.readthedocs.io/en/latest/CLI.html#openstack-keystone-authorization-environment-variables
@@ -878,4 +915,4 @@ Example with length specified:
878 915
879:: 916::
880 917
881 ./pegleg.sh generate salt -l <length> \ No newline at end of file 918 ./pegleg.sh generate salt -l <length>
diff --git a/doc/source/exceptions.rst b/doc/source/exceptions.rst
index 06585f8..24ec61b 100644
--- a/doc/source/exceptions.rst
+++ b/doc/source/exceptions.rst
@@ -68,10 +68,19 @@ PKI Exceptions
68-------------- 68--------------
69 69
70.. autoexception:: pegleg.engine.exceptions.IncompletePKIPairError 70.. autoexception:: pegleg.engine.exceptions.IncompletePKIPairError
71
72Genesis Bundle Exceptions
73-------------------------
74
75.. autoexception:: pegleg.engine.exceptions.GenesisBundleEncryptionException
71 :members: 76 :members:
72 :show-inheritance: 77 :show-inheritance:
73 :undoc-members: 78 :undoc-members:
74 79
80.. autoexception:: pegleg.engine.exceptions.GenesisBundleGenerateException
81 :members:
82 :show-inheritance:
83
75Passphrase Exceptions 84Passphrase Exceptions
76--------------------- 85---------------------
77 86
diff --git a/pegleg/cli.py b/pegleg/cli.py
index 0d93223..52a6f8b 100644
--- a/pegleg/cli.py
+++ b/pegleg/cli.py
@@ -14,13 +14,16 @@
14 14
15import functools 15import functools
16import logging 16import logging
17import os
17import sys 18import sys
18 19
19import click 20import click
20 21
21from pegleg import config 22from pegleg import config
22from pegleg import engine 23from pegleg import engine
24from pegleg.engine import bundle
23from pegleg.engine import catalog 25from pegleg.engine import catalog
26from pegleg.engine.util.pegleg_secret_management import PeglegSecretManagement
24from pegleg.engine.util.shipyard_helper import ShipyardHelper 27from pegleg.engine.util.shipyard_helper import ShipyardHelper
25 28
26LOG = logging.getLogger(__name__) 29LOG = logging.getLogger(__name__)
@@ -412,6 +415,54 @@ def generate_pki(site_name, author):
412 click.echo("Generated PKI files written to:\n%s" % '\n'.join(output_paths)) 415 click.echo("Generated PKI files written to:\n%s" % '\n'.join(output_paths))
413 416
414 417
418@site.command(
419 'genesis_bundle',
420 help='Construct the genesis deployment bundle.')
421@click.option(
422 '-b',
423 '--build-dir',
424 'build_dir',
425 type=click.Path(file_okay=False, dir_okay=True, resolve_path=True),
426 required=True,
427 help='Destination directory to store the genesis bundle.')
428@click.option(
429 '--include-validators',
430 'validators',
431 is_flag=True,
432 default=False,
433 help='A flag to request generate genesis validation scripts in addition '
434 'to genesis.sh script.')
435@SITE_REPOSITORY_ARGUMENT
436def genesis_bundle(*, build_dir, validators, site_name):
437 prom_encryption_key = os.environ.get("PROMENADE_ENCRYPTION_KEY")
438 peg_encryption_key = os.environ.get("PEGLEG_PASSPHRASE")
439 encryption_key = None
440 if (prom_encryption_key and len(prom_encryption_key) > 24 and
441 peg_encryption_key and len(peg_encryption_key) > 24):
442 click.echo("WARNING: PROMENADE_ENCRYPTION_KEY is deprecated, "
443 "using PEGLEG_PASSPHRASE instead", err=True)
444 config.set_passphrase(peg_encryption_key)
445 encryption_key = peg_encryption_key
446 elif prom_encryption_key and len(prom_encryption_key) > 24:
447 click.echo("ERROR: PROMENADE_ENCRYPTION_KEY is deprecated, "
448 "use PEGLEG_PASSPHRASE instead", err=True)
449 raise click.ClickException("ERROR: PEGLEG_PASSPHRASE must be set "
450 "and at least 24 characters long.")
451 elif peg_encryption_key and len(peg_encryption_key) > 24:
452 config.set_passphrase(peg_encryption_key)
453 encryption_key = peg_encryption_key
454 else:
455 raise click.ClickException("ERROR: PEGLEG_PASSPHRASE must be set "
456 "and at least 24 characters long.")
457
458 PeglegSecretManagement.check_environment()
459 bundle.build_genesis(build_dir,
460 encryption_key,
461 validators,
462 logging.DEBUG == LOG.getEffectiveLevel(),
463 site_name)
464
465
415@main.group(help='Commands related to types') 466@main.group(help='Commands related to types')
416@MAIN_REPOSITORY_OPTION 467@MAIN_REPOSITORY_OPTION
417@REPOSITORY_CLONE_PATH_OPTION 468@REPOSITORY_CLONE_PATH_OPTION
diff --git a/pegleg/config.py b/pegleg/config.py
index be007ca..a481e4f 100644
--- a/pegleg/config.py
+++ b/pegleg/config.py
@@ -26,7 +26,9 @@ except NameError:
26 'clone_path': None, 26 'clone_path': None,
27 'site_path': 'site', 27 'site_path': 'site',
28 'site_rev': None, 28 'site_rev': None,
29 'type_path': 'type' 29 'type_path': 'type',
30 'passphrase': None,
31 'salt': None
30 } 32 }
31 33
32 34
@@ -147,3 +149,23 @@ def set_rel_type_path(p):
147 """Set the relative type path name.""" 149 """Set the relative type path name."""
148 p = p or 'type' 150 p = p or 'type'
149 GLOBAL_CONTEXT['type_path'] = p 151 GLOBAL_CONTEXT['type_path'] = p
152
153
154def set_passphrase(p):
155 """Set the passphrase for encryption and decryption."""
156 GLOBAL_CONTEXT['passphrase'] = p
157
158
159def get_passphrase():
160 """Get the passphrase for encryption and decryption."""
161 return GLOBAL_CONTEXT['passphrase']
162
163
164def set_salt(p):
165 """Set the salt for encryption and decryption."""
166 GLOBAL_CONTEXT['salt'] = p
167
168
169def get_salt():
170 """Get the salt for encryption and decryption."""
171 return GLOBAL_CONTEXT['salt']
diff --git a/pegleg/engine/bundle.py b/pegleg/engine/bundle.py
new file mode 100644
index 0000000..c498298
--- /dev/null
+++ b/pegleg/engine/bundle.py
@@ -0,0 +1,92 @@
1# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import logging
16import os
17import stat
18
19import click
20
21from pegleg.engine.exceptions import GenesisBundleEncryptionException
22from pegleg.engine.exceptions import GenesisBundleGenerateException
23from pegleg.engine import util
24from pegleg.engine.util.pegleg_secret_management import PeglegSecretManagement
25
26from promenade.builder import Builder
27from promenade.config import Configuration
28from promenade import exceptions
29
30LOG = logging.getLogger(__name__)
31
32__all__ = [
33 'build_genesis',
34]
35
36
37def build_genesis(build_path, encryption_key, validators, debug, site_name):
38 """
39 Build the genesis deployment bundle, and store it in ``build_path``.
40
41 Build the genesis.sh script, base65-encode, encrypt and embed the
42 site configuration source documents in genesis.sh script.
43 If ``validators`` flag should be True, build the bundle validator
44 scripts as well.
45 Store the built deployment bundle in `build_path`.
46
47 :param str build_path: Directory path of the built genesis deployment
48 bundle
49 :param str encryption_key: Key to use to encrypt the bundled site
50 configuration in genesis.sh script.
51 :param bool validators: Whether to generate validator scripts
52 :param int debug: pegleg debug level to pass to promenade engine
53 for logging.
54 :return: None
55 """
56
57 # Raise an error if the build path exists. We don't want to overwrite it.
58 if os.path.isdir(build_path):
59 raise click.ClickException(
60 "{} already exists, remove it or specify a new "
61 "directory.".format(build_path))
62 # Get the list of config files
63 LOG.info('=== Building bootstrap scripts ===')
64
65 # Copy the site config, and site secrets to build directory
66 os.mkdir(build_path)
67 os.chmod(build_path, os.stat(build_path).st_mode | stat.S_IRWXU |
68 stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH)
69 documents = util.definition.documents_for_site(site_name)
70 secret_manager = PeglegSecretManagement(docs=documents)
71 documents = secret_manager.get_decrypted_secrets()
72 try:
73 # Use the promenade engine to build and encrypt the genesis bundle
74 c = Configuration(
75 documents=documents,
76 debug=debug,
77 substitute=True,
78 allow_missing_substitutions=False,
79 leave_kubectl=False)
80 if c.get_path('EncryptionPolicy:scripts.genesis') and encryption_key:
81 os.environ['PROMENADE_ENCRYPTION_KEY'] = encryption_key
82 os.environ['PEGLEG_PASSPHRASE'] = encryption_key
83 Builder(c, validators=validators).build_all(output_dir=build_path)
84 else:
85 raise GenesisBundleEncryptionException()
86
87 except exceptions.PromenadeException as e:
88 LOG.error('Build genesis bundle failed! {}.'.format(
89 e.display(debug=debug)))
90 raise GenesisBundleGenerateException()
91
92 LOG.info('=== Done! ===')
diff --git a/pegleg/engine/exceptions.py b/pegleg/engine/exceptions.py
index e26c718..d96f519 100644
--- a/pegleg/engine/exceptions.py
+++ b/pegleg/engine/exceptions.py
@@ -86,3 +86,19 @@ class PassphraseCatalogNotFoundException(PeglegBaseException):
86 """Failed to find Catalog for Passphrases generation.""" 86 """Failed to find Catalog for Passphrases generation."""
87 message = ('Could not find the Passphrase Catalog to generate ' 87 message = ('Could not find the Passphrase Catalog to generate '
88 'the site Passphrases!') 88 'the site Passphrases!')
89
90
91class GenesisBundleEncryptionException(PeglegBaseException):
92 """Exception raised when encryption of the genesis bundle fails."""
93
94 message = 'Encryption is required for genesis bundle, but no encryption ' \
95 'policy or key is specified.'
96
97
98class GenesisBundleGenerateException(PeglegBaseException):
99 """
100 Exception raised when pormenade engine fails to build the genesis
101 bundle.
102 """
103
104 message = 'Bundle generation failed on deckhand validation.'
diff --git a/pegleg/engine/util/pegleg_secret_management.py b/pegleg/engine/util/pegleg_secret_management.py
index 78e97ed..7a18589 100644
--- a/pegleg/engine/util/pegleg_secret_management.py
+++ b/pegleg/engine/util/pegleg_secret_management.py
@@ -19,6 +19,7 @@ import re
19import click 19import click
20import yaml 20import yaml
21 21
22from pegleg import config
22from pegleg.engine.util.encryption import decrypt 23from pegleg.engine.util.encryption import decrypt
23from pegleg.engine.util.encryption import encrypt 24from pegleg.engine.util.encryption import encrypt
24from pegleg.engine.util import files 25from pegleg.engine.util import files
@@ -47,10 +48,11 @@ class PeglegSecretManagement(object):
47 raise ValueError('Either `file_path` or `docs` must be ' 48 raise ValueError('Either `file_path` or `docs` must be '
48 'specified.') 49 'specified.')
49 50
50 if generated and not (author and catalog): 51 if generated and not (catalog and author):
51 raise ValueError("If the document is generated, author and " 52 raise ValueError("If the document is generated, author and "
52 "catalog must be specified.") 53 "catalog must be specified.")
53 self.__check_environment() 54
55 self.check_environment()
54 self.file_path = file_path 56 self.file_path = file_path
55 self.documents = list() 57 self.documents = list()
56 self._generated = generated 58 self._generated = generated
@@ -68,8 +70,17 @@ class PeglegSecretManagement(object):
68 70
69 self._author = author 71 self._author = author
70 72
71 self.passphrase = os.environ.get(ENV_PASSPHRASE).encode() 73 if config.get_passphrase() and config.get_salt():
72 self.salt = os.environ.get(ENV_SALT).encode() 74 self.passphrase = config.get_passphrase()
75 self.salt = config.get_salt()
76 elif config.get_passphrase() or config.get_salt():
77 raise ValueError("ERROR: Pegleg configuration must either have "
78 "both a passphrase and a salt or neither.")
79 else:
80 self.passphrase = os.environ.get(ENV_PASSPHRASE).encode()
81 self.salt = os.environ.get(ENV_SALT).encode()
82 config.set_passphrase(self.passphrase)
83 config.set_salt(self.salt)
73 84
74 def __iter__(self): 85 def __iter__(self):
75 """ 86 """
@@ -79,7 +90,7 @@ class PeglegSecretManagement(object):
79 return (doc.pegleg_document for doc in self.documents) 90 return (doc.pegleg_document for doc in self.documents)
80 91
81 @staticmethod 92 @staticmethod
82 def __check_environment(): 93 def check_environment():
83 """ 94 """
84 Validate required environment variables for encryption or decryption. 95 Validate required environment variables for encryption or decryption.
85 96
diff --git a/requirements.txt b/requirements.txt
index 41d4561..4f97c4c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,3 +9,4 @@ python-dateutil==2.7.3
9rstr==2.2.6 9rstr==2.2.6
10git+https://github.com/openstack/airship-deckhand.git@49ad9f38842f7f1ecb86d907d86d332f8186eb8c 10git+https://github.com/openstack/airship-deckhand.git@49ad9f38842f7f1ecb86d907d86d332f8186eb8c
11git+https://github.com/openstack/airship-shipyard.git@44f7022df6438de541501c2fdd5c46df198b82bf#egg=shipyard_client&subdirectory=src/bin/shipyard_client 11git+https://github.com/openstack/airship-shipyard.git@44f7022df6438de541501c2fdd5c46df198b82bf#egg=shipyard_client&subdirectory=src/bin/shipyard_client
12git+https://github.com/openstack/airship-promenade.git@a6e8fdbe22bd153c78a008b92cd5d1c245bc63e3
diff --git a/tests/unit/engine/test_build_genesis_bundle.py b/tests/unit/engine/test_build_genesis_bundle.py
new file mode 100644
index 0000000..21baba9
--- /dev/null
+++ b/tests/unit/engine/test_build_genesis_bundle.py
@@ -0,0 +1,144 @@
1# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import logging
16import os
17
18import mock
19import pytest
20import yaml
21
22from pegleg import config
23from pegleg.engine import bundle
24from pegleg.engine.exceptions import GenesisBundleEncryptionException
25from pegleg.engine.exceptions import GenesisBundleGenerateException
26from pegleg.engine.util import files
27from pegleg.engine.util.pegleg_secret_management import ENV_PASSPHRASE
28from pegleg.engine.util.pegleg_secret_management import ENV_SALT
29
30from tests.unit.fixtures import temp_path
31
32SITE_DEFINITION = """
33---
34# High-level pegleg site definition file
35schema: pegleg/SiteDefinition/v1
36metadata:
37 schema: metadata/Document/v1
38 layeringDefinition:
39 abstract: false
40 layer: site
41 # NEWSITE-CHANGEME: Replace with the site name
42 name: test_site
43 storagePolicy: cleartext
44data:
45 # The type layer this site will delpoy with. Type layer is found in the
46 # type folder.
47 site_type: foundry
48...
49
50"""
51
52SITE_CONFIG_DATA = """
53---
54schema: promenade/EncryptionPolicy/v1
55metadata:
56 schema: metadata/Document/v1
57 name: encryption-policy
58 layeringDefinition:
59 abstract: false
60 layer: site
61 storagePolicy: cleartext
62data:
63 scripts:
64 genesis:
65 gpg: {}
66 join:
67 gpg: {}
68---
69schema: deckhand/LayeringPolicy/v1
70metadata:
71 schema: metadata/Control/v1
72 name: layering-policy
73data:
74 layerOrder:
75 - global
76 - type
77 - site
78---
79schema: deckhand/Passphrase/v1
80metadata:
81 schema: metadata/Document/v1
82 name: ceph_swift_keystone_password
83 layeringDefinition:
84 abstract: false
85 layer: site
86 storagePolicy: cleartext
87data: ABAgagajajkb839215387
88...
89"""
90
91
92@mock.patch.dict(os.environ, {
93 ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
94 ENV_SALT: 'MySecretSalt'
95})
96def test_no_encryption_key(temp_path):
97 # Write the test data to temp file
98 config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA))
99 base_config_dir = os.path.join(temp_path, 'config_dir')
100 config.set_site_repo(base_config_dir)
101 config_dir = os.path.join(base_config_dir, 'site', 'test_site')
102
103 config_path = os.path.join(config_dir, 'config_file.yaml')
104 build_dir = os.path.join(temp_path, 'build_dir')
105 os.makedirs(config_dir)
106
107 files.write(config_path, config_data)
108 files.write(os.path.join(config_dir, "site-definition.yaml"),
109 yaml.safe_load_all(SITE_DEFINITION))
110
111 with pytest.raises(GenesisBundleEncryptionException,
112 match=r'.*no encryption policy or key is specified.*'):
113 bundle.build_genesis(build_path=build_dir,
114 encryption_key=None,
115 validators=False,
116 debug=logging.ERROR,
117 site_name="test_site")
118
119
120@mock.patch.dict(os.environ, {
121 ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
122 ENV_SALT: 'MySecretSalt'
123})
124def test_failed_deckhand_validation(temp_path):
125 # Write the test data to temp file
126 config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA))
127 base_config_dir = os.path.join(temp_path, 'config_dir')
128 config.set_site_repo(base_config_dir)
129 config_dir = os.path.join(base_config_dir, 'site', 'test_site')
130
131 config_path = os.path.join(config_dir, 'config_file.yaml')
132 build_dir = os.path.join(temp_path, 'build_dir')
133 os.makedirs(config_dir)
134 files.write(config_path, config_data)
135 files.write(os.path.join(config_dir, "site-definition.yaml"),
136 yaml.safe_load_all(SITE_DEFINITION))
137 key = 'MyverYSecretEncryptionKey382803'
138 with pytest.raises(GenesisBundleGenerateException,
139 match=r'.*failed on deckhand validation.*'):
140 bundle.build_genesis(build_path=build_dir,
141 encryption_key=key,
142 validators=False,
143 debug=logging.ERROR,
144 site_name="test_site")