From 8bc8c7c02856c27840b6e17bc20d004e127135d5 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Thu, 23 Aug 2018 15:56:31 -0500 Subject: [PATCH] Implement encryption for genesis/join scripts This introduces a new document called `EncryptionPolicy` to configure this behavior. It currently only supports using symmetric encryption with `GPG`, but that should be available on all Ubuntu systems (which is what we currently support) and should also be fairly reliable. Change-Id: I06d4faa119b736773df0d8cbf0e7a23fd98edcdf Depends-On: https://review.openstack.org/#/c/602175/ --- Makefile | 8 +- .../configuration/encryption-policy.yaml | 33 + doc/source/configuration/index.rst | 1 + examples/basic/EncryptionPolicy.yaml | 14 + promenade/builder.py | 60 +- promenade/encryption_method.py | 182 +++ promenade/exceptions.py | 10 + promenade/generator.py | 21 +- promenade/pki.py | 19 +- promenade/schemas/EncryptionPolicy.yaml | 33 + promenade/templates/include/up.sh | 5 +- promenade/validation.py | 2 +- tests/unit/builder_data/simple/Docker.yaml | 18 + .../builder_data/simple/EncryptionPolicy.yaml | 16 + tests/unit/builder_data/simple/Genesis.yaml | 45 + .../unit/builder_data/simple/HostSystem.yaml | 86 ++ tests/unit/builder_data/simple/Kubelet.yaml | 21 + .../simple/KubernetesNetwork.yaml | 43 + .../builder_data/simple/KubernetesNode.yaml | 31 + .../builder_data/simple/LayeringPolicy.yaml | 11 + .../unit/builder_data/simple/PKICatalog.yaml | 167 +++ .../builder_data/simple/armada-resources.yaml | 1061 +++++++++++++++++ tests/unit/test_builder.py | 57 + tools/g2/lib/config.sh | 1 + tools/g2/stages/build-scripts.sh | 1 + tools/g2/stages/genesis.sh | 2 +- tools/install-external-deps.sh | 16 + tools/lint_gate.sh | 2 +- 28 files changed, 1936 insertions(+), 30 deletions(-) create mode 100644 doc/source/configuration/encryption-policy.yaml create mode 100644 examples/basic/EncryptionPolicy.yaml create mode 100644 promenade/encryption_method.py create mode 100644 promenade/schemas/EncryptionPolicy.yaml create mode 100644 tests/unit/builder_data/simple/Docker.yaml create mode 100644 tests/unit/builder_data/simple/EncryptionPolicy.yaml create mode 100644 tests/unit/builder_data/simple/Genesis.yaml create mode 100644 tests/unit/builder_data/simple/HostSystem.yaml create mode 100644 tests/unit/builder_data/simple/Kubelet.yaml create mode 100644 tests/unit/builder_data/simple/KubernetesNetwork.yaml create mode 100644 tests/unit/builder_data/simple/KubernetesNode.yaml create mode 100644 tests/unit/builder_data/simple/LayeringPolicy.yaml create mode 100644 tests/unit/builder_data/simple/PKICatalog.yaml create mode 100644 tests/unit/builder_data/simple/armada-resources.yaml create mode 100644 tests/unit/test_builder.py create mode 100755 tools/install-external-deps.sh diff --git a/Makefile b/Makefile index a71b9d97..cceb9497 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ CHARTS := $(patsubst charts/%/.,%,$(wildcard charts/*/.)) all: charts lint .PHONY: tests -tests: gate-lint +tests: external-deps gate-lint tox .PHONY: tests-security @@ -48,9 +48,13 @@ docs: tox -e docs .PHONY: tests-unit -tests-unit: +tests-unit: external-deps tox -e py35 +.PHONY: external-deps +external-deps: + ./tools/install-external-deps.sh + .PHONY: tests-pep8 tests-pep8: tox -e pep8 diff --git a/doc/source/configuration/encryption-policy.yaml b/doc/source/configuration/encryption-policy.yaml new file mode 100644 index 00000000..423978ce --- /dev/null +++ b/doc/source/configuration/encryption-policy.yaml @@ -0,0 +1,33 @@ +EncryptionPolicy +================ + +Encryption policy defines how encryption should be applied via Promenade. The +primary use-case for this is to encrypt ``genesis.sh`` or ``join.sh`` scripts. + +Sample Document +--------------- + +.. code-block:: yaml + + --- + schema: promenade/EncryptionPolicy/v1 + metadata: + schema: metadata/Document/v1 + name: encryption-policy + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext + data: + scripts: + genesis: + gpg: {} + ... + + +Scripts +------- + +The genesis and join scripts can be built with sensitive content encrypted. +Currently the only encryption method available is ``gpg``, which can be enabled +by setting that key to an empty dictionary. diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index cffb630c..9317b606 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -12,6 +12,7 @@ Details about Promenade-specific documents can be found here: :caption: Documents docker + encryption-policy genesis host-system kubelet diff --git a/examples/basic/EncryptionPolicy.yaml b/examples/basic/EncryptionPolicy.yaml new file mode 100644 index 00000000..a7b66ce1 --- /dev/null +++ b/examples/basic/EncryptionPolicy.yaml @@ -0,0 +1,14 @@ +--- +schema: promenade/EncryptionPolicy/v1 +metadata: + schema: metadata/Document/v1 + name: encryption-policy + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + scripts: + genesis: + gpg: {} +... diff --git a/promenade/builder.py b/promenade/builder.py index 49d210a4..ff5cd6c2 100644 --- a/promenade/builder.py +++ b/promenade/builder.py @@ -1,4 +1,4 @@ -from . import logging, renderer +from . import encryption_method, logging, renderer from beaker.cache import CacheManager from beaker.util import parse_cache_config_options @@ -72,6 +72,14 @@ class Builder: _write_script(output_dir, 'validate-cluster.sh', validate_script) def build_genesis(self, *, output_dir): + script = self.build_genesis_script() + _write_script(output_dir, 'genesis.sh', script) + + if self.validators: + validate_script = self._build_genesis_validate_script() + _write_script(output_dir, 'validate-genesis.sh', validate_script) + + def build_genesis_script(self): LOG.info('Building genesis script') sub_config = self.config.extract_genesis_config() tarball = renderer.build_tarball_from_roles( @@ -79,17 +87,23 @@ class Builder: roles=['common', 'genesis'], file_specs=self.file_cache.values()) - script = renderer.render_template( + (encrypted_tarball, decrypt_setup_command, decrypt_command, + decrypt_teardown_command) = _encrypt_genesis(sub_config, tarball) + + return renderer.render_template( sub_config, template='scripts/genesis.sh', - context={'tarball': tarball}) + context={ + 'decrypt_command': decrypt_command, + 'decrypt_setup_command': decrypt_setup_command, + 'decrypt_teardown_command': decrypt_teardown_command, + 'encrypted_tarball': encrypted_tarball, + }) - _write_script(output_dir, 'genesis.sh', script) - - if self.validators: - validate_script = renderer.render_template( - sub_config, template='scripts/validate-genesis.sh') - _write_script(output_dir, 'validate-genesis.sh', validate_script) + def _build_genesis_validate_script(self): + sub_config = self.config.extract_genesis_config() + return renderer.render_template( + sub_config, template='scripts/validate-genesis.sh') def build_node(self, node_document, *, output_dir): node_name = node_document['metadata']['name'] @@ -112,10 +126,18 @@ class Builder: tarball = renderer.build_tarball_from_roles( config=sub_config, roles=['common', 'join'], file_specs=file_specs) + (encrypted_tarball, decrypt_setup_command, decrypt_command, + decrypt_teardown_command) = _encrypt_node(sub_config, tarball) + return renderer.render_template( sub_config, template='scripts/join.sh', - context={'tarball': tarball}) + context={ + 'decrypt_command': decrypt_command, + 'decrypt_setup_command': decrypt_setup_command, + 'decrypt_teardown_command': decrypt_teardown_command, + 'encrypted_tarball': encrypted_tarball, + }) def _build_node_validate_script(self, node_name): sub_config = self.config.extract_node_config(node_name) @@ -123,6 +145,24 @@ class Builder: sub_config, template='scripts/validate-join.sh') +def _encrypt_genesis(config, data): + return _encrypt(config.get_path('EncryptionPolicy:scripts.genesis'), data) + + +def _encrypt_node(config, data): + return _encrypt(config.get_path('EncryptionPolicy:scripts.join'), data) + + +def _encrypt(cfg_dict, data): + method = encryption_method.EncryptionMethod.from_config(cfg_dict) + encrypted_data = method.encrypt(data) + decrypt_setup_command = method.get_decrypt_setup_command() + decrypt_command = method.get_decrypt_command() + decrypt_teardown_command = method.get_decrypt_teardown_command() + return (encrypted_data, decrypt_setup_command, decrypt_command, + decrypt_teardown_command) + + @CACHE.cache('fetch_tarball_content', expire=72 * 3600) def _fetch_tar_content(url, path): content = _fetch_tar_url(url) diff --git a/promenade/encryption_method.py b/promenade/encryption_method.py new file mode 100644 index 00000000..bca35a97 --- /dev/null +++ b/promenade/encryption_method.py @@ -0,0 +1,182 @@ +from . import exceptions, logging +import abc +import os +# Ignore bandit false positive: B404:blacklist +# The purpose of this module is to safely encapsulate calls via fork. +import subprocess # nosec +import tempfile + +__all__ = ['EncryptionMethod'] + +LOG = logging.getLogger(__name__) + + +class EncryptionMethod(metaclass=abc.ABCMeta): + @abc.abstractmethod + def encrypt(self, data): + pass + + @abc.abstractmethod + def get_decrypt_setup_command(self): + pass + + @abc.abstractmethod + def get_decrypt_command(self): + pass + + @abc.abstractmethod + def get_decrypt_teardown_command(self): + pass + + @staticmethod + def from_config(config): + LOG.debug('Building EncryptionMethod from: %s', config) + if config: + # NOTE(mark-burnett): Relying on the schema to ensure valid + # configuration. + name = list(config.keys())[0] + kwargs = config[name] + if name == 'gpg': + return GPGEncryptionMethod(**kwargs) + else: + raise NotImplementedError('Unknown Encryption method') + else: + return NullEncryptionMethod() + + def notify_user(self, message): + print('=== BEGIN NOTICE ===') + print(message) + print('=== END NOTICE ===') + + +class NullEncryptionMethod(EncryptionMethod): + def encrypt(self, data): + LOG.debug('Performing NOOP encryption') + return data + + def get_decrypt_setup_command(self): + return '' + + def get_decrypt_command(self): + return 'cat' + + def get_decrypt_teardown_command(self): + return '' + + +class GPGEncryptionMethod(EncryptionMethod): + ENCRYPTION_KEY_ENV_NAME = 'PROMENADE_ENCRYPTION_KEY' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._gpg_version = _detect_gpg_version() + + def encrypt(self, data): + key = self._get_key() + return self._encrypt_data(key, data) + + def get_decrypt_setup_command(self): + return ''' + export DECRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY:-"NONE"} + if [[ ${PROMENADE_ENCRYPTION_KEY} = "NONE" ]]; then + read -p "Script decryption key: " -s DECRYPTION_KEY + fi + ''' + + def get_decrypt_command(self): + return ('/usr/bin/gpg --verbose --decrypt ' + '--passphrase "${DECRYPTION_KEY}"') + + def get_decrypt_teardown_command(self): + return 'unset DECRYPTION_KEY' + + def _get_key(self): + key = os.environ.get(self.ENCRYPTION_KEY_ENV_NAME) + if key is None: + key = _generate_key() + self.notify_user('Copy this decryption key for use during script ' + 'execution:\n%s' % key) + else: + LOG.info('Using encryption key from %s', + self.ENCRYPTION_KEY_ENV_NAME) + + return key + + def _encrypt_data(self, key, data): + with tempfile.TemporaryDirectory() as tmp: + # Ignore bandit false positive: + # B603:subprocess_without_shell_equals_true + # Here user input is allowed to be arbitrary, as it's simply input + # to the specified encryption algorithm. Regardless, we only put a + # tarball here. + p = subprocess.Popen( # nosec + [ + '/usr/bin/gpg', + '--verbose', + '--symmetric', + '--homedir', + tmp, + '--passphrase', + key, + ] + self._gpg_encrypt_options(), + cwd=tmp, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + try: + out, err = p.communicate(data, timeout=120) + except subprocess.TimeoutExpired: + p.kill() + out, err = p.communicate() + + if p.returncode != 0: + LOG.error('Got errors from gpg encrypt: %s', err) + raise exceptions.EncryptionException(description=str(err)) + + return out + + def _gpg_encrypt_options(self): + options = { + 1: [], + 2: ['--pinentry-mode', 'loopback'], + } + return options[self._gpg_version[0]] + + +DETECTION_PREFIX = 'gpg (GnuPG) ' + + +def _detect_gpg_version(): + with tempfile.TemporaryDirectory() as tmp: + # Ignore bandit false positive: + # B603:subprocess_without_shell_equals_true + # This method takes no input and simply queries the version of gpg. + output = subprocess.check_output( # nosec + [ + '/usr/bin/gpg', + '--version', + ], cwd=tmp) + lines = output.decode('utf-8').strip().splitlines() + if lines: + version = lines[0][len(DETECTION_PREFIX):] + LOG.debug('Found GPG version %s', version) + return tuple(map(int, version.split('.')[:2])) + else: + raise exceptions.GPGDetectionException() + + +def _generate_key(): + # Ignore bandit false positive: + # B603:subprocess_without_shell_equals_true + # This method takes no input and generates random output. + result = subprocess.run( # nosec + ['/usr/bin/openssl', 'rand', '-hex', '48'], + check=True, + env={ + 'RANDFILE': '/tmp/rnd', + }, + stdout=subprocess.PIPE, + ) + + return result.stdout.decode().strip() diff --git a/promenade/exceptions.py b/promenade/exceptions.py index 2a18cbb2..abf4a231 100644 --- a/promenade/exceptions.py +++ b/promenade/exceptions.py @@ -343,6 +343,16 @@ class NodeNotFoundException(KubernetesApiError): status = falcon.HTTP_404 +class EncryptionException(ApiError): + title = 'Payload encryption error' + status = falcon.HTTP_500 + + +class GPGDetectionException(ApiError): + title = 'Failed to detect GPG version' + status = falcon.HTTP_500 + + def massage_error_list(error_list, placeholder_description): """ Returns a best-effort attempt to make a nice error list diff --git a/promenade/generator.py b/promenade/generator.py index cb470192..4204281f 100644 --- a/promenade/generator.py +++ b/promenade/generator.py @@ -10,17 +10,16 @@ LOG = logging.getLogger(__name__) class Generator: - def __init__(self, config): + def __init__(self, config, block_strings=True): self.config = config - self.keys = pki.PKI() - self.documents = [] + self.keys = pki.PKI(block_strings=block_strings) self.outputs = collections.defaultdict(dict) @property def cluster_domain(self): return self.config['KubernetesNetwork:dns.cluster_domain'] - def generate(self, output_dir): + def generate(self, output_dir=None): for catalog in self.config.iterate(kind='PKICatalog'): for ca_name, ca_def in catalog['data'].get( 'certificate_authorities', {}).items(): @@ -40,7 +39,8 @@ class Generator: document_name = keypair_def['name'] self.get_or_gen_keypair(document_name) - self._write(output_dir) + if output_dir: + self._write(output_dir) def get_or_gen_ca(self, document_name): kinds = [ @@ -126,18 +126,21 @@ class Generator: return result def _write(self, output_dir): - docs = list( - itertools.chain.from_iterable( - v.values() for v in self.outputs.values())) + documents = self.get_documents() with open(os.path.join(output_dir, 'certificates.yaml'), 'w') as f: # Don't use safe_dump_all so we can block format certificate data. yaml.dump_all( - docs, + documents, stream=f, default_flow_style=False, explicit_start=True, indent=2) + def get_documents(self): + return list( + itertools.chain.from_iterable( + v.values() for v in self.outputs.values())) + def get_host_list(service_names): service_list = [] diff --git a/promenade/pki.py b/promenade/pki.py index 025a0ac0..605c3284 100644 --- a/promenade/pki.py +++ b/promenade/pki.py @@ -13,7 +13,8 @@ LOG = logging.getLogger(__name__) class PKI: - def __init__(self): + def __init__(self, *, block_strings=True): + self.block_strings = block_strings self._ca_config_string = None @property @@ -116,9 +117,11 @@ class PKI: # Ignore bandit false positive: # B603:subprocess_without_shell_equals_true # This method wraps cfssl calls originating from this module. - return json.loads( # nosec - subprocess.check_output( - ['cfssl'] + command, cwd=tmp, stderr=subprocess.PIPE)) + result = subprocess.check_output( # nosec + ['cfssl'] + command, cwd=tmp, stderr=subprocess.PIPE) + if not isinstance(result, str): + result = result.decode('utf-8') + return json.loads(result) def _openssl(self, command, *, files=None): if not files: @@ -175,9 +178,15 @@ class PKI: }, 'storagePolicy': 'cleartext', }, - 'data': block_literal(data), + 'data': self._block_literal(data), } + def _block_literal(self, data): + if self.block_strings: + return block_literal(data) + else: + return data + class block_literal(str): pass diff --git a/promenade/schemas/EncryptionPolicy.yaml b/promenade/schemas/EncryptionPolicy.yaml new file mode 100644 index 00000000..3a1d9aca --- /dev/null +++ b/promenade/schemas/EncryptionPolicy.yaml @@ -0,0 +1,33 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: promenade/EncryptionPolicy/v1 + labels: + application: promenade +data: + $schema: http://json-schema.org/schema# + + definitions: + script_encryption: + oneof: + - { $ref: '#/definitions/encryption_method_gpg' } + + encryption_method_gpg: + properties: + gpg: + type: object + additionalProperties: false + required: + - gpg + additionalProperties: false + + properties: + scripts: + properties: + genesis: + $ref: '#/definitions/script_encryption' + join: + $ref: '#/definitions/script_encryption' + additionalProperties: false +... diff --git a/promenade/templates/include/up.sh b/promenade/templates/include/up.sh index 2574e808..22c87c34 100644 --- a/promenade/templates/include/up.sh +++ b/promenade/templates/include/up.sh @@ -10,7 +10,10 @@ chmod 700 /etc/kubernetes set +x log log === Extracting prepared files === -echo "{{ tarball | b64enc }}" | base64 -d | tar -zxv -C / | tee /etc/promenade-manifest +{{ decrypt_setup_command }} +echo "{{ encrypted_tarball | b64enc }}" | base64 -d | {{ decrypt_command }} | tar -zxv -C / | tee /etc/promenade-manifest +{{ decrypt_teardown_command }} +set -x # Adding apt repositories # diff --git a/promenade/validation.py b/promenade/validation.py index d78823be..5a111dc8 100644 --- a/promenade/validation.py +++ b/promenade/validation.py @@ -105,7 +105,7 @@ def check_schema(document, schemas=None): except jsonschema.ValidationError as e: raise exceptions.ValidationException(str(e)) else: - LOG.warning('Skipping validation for unknown schema: %s', schema_name) + LOG.debug('Skipping validation for unknown schema: %s', schema_name) SCHEMAS = {} diff --git a/tests/unit/builder_data/simple/Docker.yaml b/tests/unit/builder_data/simple/Docker.yaml new file mode 100644 index 00000000..9463e9f9 --- /dev/null +++ b/tests/unit/builder_data/simple/Docker.yaml @@ -0,0 +1,18 @@ +--- +schema: promenade/Docker/v1 +metadata: + schema: metadata/Document/v1 + name: docker + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + config: + insecure-registries: + - registry:5000 + live-restore: true + max-concurrent-downloads: 10 + oom-score-adjust: -999 + storage-driver: overlay2 +... diff --git a/tests/unit/builder_data/simple/EncryptionPolicy.yaml b/tests/unit/builder_data/simple/EncryptionPolicy.yaml new file mode 100644 index 00000000..c17bc27d --- /dev/null +++ b/tests/unit/builder_data/simple/EncryptionPolicy.yaml @@ -0,0 +1,16 @@ +--- +schema: promenade/EncryptionPolicy/v1 +metadata: + schema: metadata/Document/v1 + name: testingpolicy + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + scripts: + genesis: + gpg: {} + join: + gpg: {} +... diff --git a/tests/unit/builder_data/simple/Genesis.yaml b/tests/unit/builder_data/simple/Genesis.yaml new file mode 100644 index 00000000..bb67666f --- /dev/null +++ b/tests/unit/builder_data/simple/Genesis.yaml @@ -0,0 +1,45 @@ +--- +schema: promenade/Genesis/v1 +metadata: + schema: metadata/Document/v1 + name: genesis + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + hostname: n0 + ip: 192.168.77.10 + apiserver: + command_prefix: + - /apiserver + - --authorization-mode=Node,RBAC + - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds + - --service-cluster-ip-range=10.96.0.0/16 + - --endpoint-reconciler-type=lease + armada: + target_manifest: cluster-bootstrap + labels: + dynamic: + - calico-etcd=enabled + - coredns=enabled + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - promenade-genesis=enabled + - ucp-control-plane=enabled + images: + armada: quay.io/airshipit/armada:master + helm: + tiller: gcr.io/kubernetes-helm/tiller:v2.9.1 + kubernetes: + apiserver: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + controller-manager: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + etcd: quay.io/coreos/etcd:v3.2.14 + scheduler: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + files: + - path: /var/lib/anchor/calico-etcd-bootstrap + content: "# placeholder for triggering calico etcd bootstrapping" + mode: 0644 +... diff --git a/tests/unit/builder_data/simple/HostSystem.yaml b/tests/unit/builder_data/simple/HostSystem.yaml new file mode 100644 index 00000000..c38c899e --- /dev/null +++ b/tests/unit/builder_data/simple/HostSystem.yaml @@ -0,0 +1,86 @@ +--- +schema: promenade/HostSystem/v1 +metadata: + schema: metadata/Document/v1 + name: host-system + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + files: + # NOTE(mark-burnett): A kubelet would be required for a real deployment + # (either here or via debian package; however, these unit tests don't + # attempt to actually run Kubernetes, only to construct the genesis and + # join scripts. + # - path: /opt/kubernetes/bin/kubelet + # tar_url: https://dl.k8s.io/v1.10.2/kubernetes-node-linux-amd64.tar.gz + # tar_path: kubernetes/node/bin/kubelet + # mode: 0555 + - path: /etc/logrotate.d/json-logrotate + mode: 0444 + content: |- + /var/lib/docker/containers/*/*-json.log + { + compress + copytruncate + create 0644 root root + daily + dateext + dateformat -%Y%m%d-%s + maxsize 10M + missingok + notifempty + su root root + rotate 1 + } + images: + haproxy: haproxy:1.8.3 + helm: + helm: lachlanevenson/k8s-helm:v2.9.1 + kubernetes: + kubectl: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + packages: + repositories: + - deb http://apt.dockerproject.org/repo ubuntu-xenial main + keys: + - |- + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFWln24BEADrBl5p99uKh8+rpvqJ48u4eTtjeXAWbslJotmC/CakbNSqOb9o + ddfzRvGVeJVERt/Q/mlvEqgnyTQy+e6oEYN2Y2kqXceUhXagThnqCoxcEJ3+KM4R + mYdoe/BJ/J/6rHOjq7Omk24z2qB3RU1uAv57iY5VGw5p45uZB4C4pNNsBJXoCvPn + TGAs/7IrekFZDDgVraPx/hdiwopQ8NltSfZCyu/jPpWFK28TR8yfVlzYFwibj5WK + dHM7ZTqlA1tHIG+agyPf3Rae0jPMsHR6q+arXVwMccyOi+ULU0z8mHUJ3iEMIrpT + X+80KaN/ZjibfsBOCjcfiJSB/acn4nxQQgNZigna32velafhQivsNREFeJpzENiG + HOoyC6qVeOgKrRiKxzymj0FIMLru/iFF5pSWcBQB7PYlt8J0G80lAcPr6VCiN+4c + NKv03SdvA69dCOj79PuO9IIvQsJXsSq96HB+TeEmmL+xSdpGtGdCJHHM1fDeCqkZ + hT+RtBGQL2SEdWjxbF43oQopocT8cHvyX6Zaltn0svoGs+wX3Z/H6/8P5anog43U + 65c0A+64Jj00rNDr8j31izhtQMRo892kGeQAaaxg4Pz6HnS7hRC+cOMHUU4HA7iM + zHrouAdYeTZeZEQOA7SxtCME9ZnGwe2grxPXh/U/80WJGkzLFNcTKdv+rwARAQAB + tDdEb2NrZXIgUmVsZWFzZSBUb29sIChyZWxlYXNlZG9ja2VyKSA8ZG9ja2VyQGRv + Y2tlci5jb20+iQI4BBMBAgAiBQJVpZ9uAhsvBgsJCAcDAgYVCAIJCgsEFgIDAQIe + AQIXgAAKCRD3YiFXLFJgnbRfEAC9Uai7Rv20QIDlDogRzd+Vebg4ahyoUdj0CH+n + Ak40RIoq6G26u1e+sdgjpCa8jF6vrx+smpgd1HeJdmpahUX0XN3X9f9qU9oj9A4I + 1WDalRWJh+tP5WNv2ySy6AwcP9QnjuBMRTnTK27pk1sEMg9oJHK5p+ts8hlSC4Sl + uyMKH5NMVy9c+A9yqq9NF6M6d6/ehKfBFFLG9BX+XLBATvf1ZemGVHQusCQebTGv + 0C0V9yqtdPdRWVIEhHxyNHATaVYOafTj/EF0lDxLl6zDT6trRV5n9F1VCEh4Aal8 + L5MxVPcIZVO7NHT2EkQgn8CvWjV3oKl2GopZF8V4XdJRl90U/WDv/6cmfI08GkzD + YBHhS8ULWRFwGKobsSTyIvnbk4NtKdnTGyTJCQ8+6i52s+C54PiNgfj2ieNn6oOR + 7d+bNCcG1CdOYY+ZXVOcsjl73UYvtJrO0Rl/NpYERkZ5d/tzw4jZ6FCXgggA/Zxc + jk6Y1ZvIm8Mt8wLRFH9Nww+FVsCtaCXJLP8DlJLASMD9rl5QS9Ku3u7ZNrr5HWXP + HXITX660jglyshch6CWeiUATqjIAzkEQom/kEnOrvJAtkypRJ59vYQOedZ1sFVEL + MXg2UCkD/FwojfnVtjzYaTCeGwFQeqzHmM241iuOmBYPeyTY5veF49aBJA1gEJOQ + TvBR8Q== + =Fm3p + -----END PGP PUBLIC KEY BLOCK----- + additional: + - curl + - jq + required: + docker: docker-engine=1.13.1-0~ubuntu-xenial + socat: socat=1.7.3.1-1 + validation: + pod_logs: + image: busybox:1.28.3 +... diff --git a/tests/unit/builder_data/simple/Kubelet.yaml b/tests/unit/builder_data/simple/Kubelet.yaml new file mode 100644 index 00000000..41b84ce4 --- /dev/null +++ b/tests/unit/builder_data/simple/Kubelet.yaml @@ -0,0 +1,21 @@ +--- +schema: promenade/Kubelet/v1 +metadata: + schema: metadata/Document/v1 + name: kubelet + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + arguments: + - --cni-bin-dir=/opt/cni/bin + - --cni-conf-dir=/etc/cni/net.d + - --eviction-max-pod-grace-period=-1 + - --network-plugin=cni + - --node-status-update-frequency=5s + - --serialize-image-pulls=false + - --v=5 + images: + pause: gcr.io/google_containers/pause-amd64:3.0 +... diff --git a/tests/unit/builder_data/simple/KubernetesNetwork.yaml b/tests/unit/builder_data/simple/KubernetesNetwork.yaml new file mode 100644 index 00000000..1e35dbb8 --- /dev/null +++ b/tests/unit/builder_data/simple/KubernetesNetwork.yaml @@ -0,0 +1,43 @@ +--- +schema: promenade/KubernetesNetwork/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-network + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + dns: + cluster_domain: cluster.local + service_ip: 10.96.0.10 + bootstrap_validation_checks: + - calico-etcd.kube-system.svc.cluster.local + - google.com + - kubernetes-etcd.kube-system.svc.cluster.local + - kubernetes.default.svc.cluster.local + upstream_servers: + - 8.8.8.8 + - 8.8.4.4 + + kubernetes: + apiserver_port: 6443 + haproxy_port: 6553 + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + service_ip: 10.96.0.1 + + etcd: + container_port: 2379 + haproxy_port: 2378 + + hosts_entries: + - ip: 192.168.77.1 + names: + - registry + +# proxy: +# url: http://proxy.example.com:8080 +# additional_no_proxy: +# - 10.0.1.1 +... diff --git a/tests/unit/builder_data/simple/KubernetesNode.yaml b/tests/unit/builder_data/simple/KubernetesNode.yaml new file mode 100644 index 00000000..325b4412 --- /dev/null +++ b/tests/unit/builder_data/simple/KubernetesNode.yaml @@ -0,0 +1,31 @@ +--- +schema: promenade/KubernetesNode/v1 +metadata: + schema: metadata/Document/v1 + name: n1 + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + hostname: n1 + ip: 192.169.77.11 + join_ip: 192.169.77.10 + labels: + dynamic: + - calico-etcd=enabled + - ceph-mds=enabled + - ceph-mon=enabled + - ceph-osd=enabled + - ceph-rgw=enabled + - ceph-mgr=enabled + - coredns=enabled + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - openstack-compute-node=enabled + - openstack-control-plane=enabled + - openvswitch=enabled + - ucp-control-plane=enabled +... diff --git a/tests/unit/builder_data/simple/LayeringPolicy.yaml b/tests/unit/builder_data/simple/LayeringPolicy.yaml new file mode 100644 index 00000000..46ae0c58 --- /dev/null +++ b/tests/unit/builder_data/simple/LayeringPolicy.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/LayeringPolicy/v1 +metadata: + schema: metadata/Control/v1 + name: layering-policy +data: + layerOrder: + - global + - type + - site +... diff --git a/tests/unit/builder_data/simple/PKICatalog.yaml b/tests/unit/builder_data/simple/PKICatalog.yaml new file mode 100644 index 00000000..48b88365 --- /dev/null +++ b/tests/unit/builder_data/simple/PKICatalog.yaml @@ -0,0 +1,167 @@ +--- +schema: promenade/PKICatalog/v1 +metadata: + schema: metadata/Document/v1 + name: cluster-certificates + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + certificate_authorities: + kubernetes: + description: CA for Kubernetes components + certificates: + - document_name: apiserver + description: Service certificate for Kubernetes apiserver + common_name: apiserver + hosts: + - localhost + - 127.0.0.1 + - 10.96.0.1 + kubernetes_service_names: + - kubernetes.default.svc.cluster.local + - document_name: kubelet-genesis + common_name: system:node:n0 + hosts: + - n0 + - 192.168.77.10 + groups: + - system:nodes + - document_name: kubelet-n0 + common_name: system:node:n0 + hosts: + - n0 + - 192.168.77.10 + groups: + - system:nodes + - document_name: kubelet-n1 + common_name: system:node:n1 + hosts: + - n1 + - 192.168.77.11 + groups: + - system:nodes + - document_name: scheduler + description: Service certificate for Kubernetes scheduler + common_name: system:kube-scheduler + - document_name: controller-manager + description: certificate for controller-manager + common_name: system:kube-controller-manager + - document_name: admin + common_name: admin + groups: + - system:masters + - document_name: armada + common_name: armada + groups: + - system:masters + kubernetes-etcd: + description: Certificates for Kubernetes's etcd servers + certificates: + - document_name: apiserver-etcd + description: etcd client certificate for use by Kubernetes apiserver + common_name: apiserver + # NOTE(mark-burnett): hosts not required for client certificates + - document_name: kubernetes-etcd-anchor + description: anchor + common_name: anchor + - document_name: kubernetes-etcd-genesis + common_name: kubernetes-etcd-genesis + hosts: + - n0 + - 192.168.77.10 + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - document_name: kubernetes-etcd-n0 + common_name: kubernetes-etcd-n0 + hosts: + - n0 + - 192.168.77.10 + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - document_name: kubernetes-etcd-n1 + common_name: kubernetes-etcd-n1 + hosts: + - n1 + - 192.168.77.11 + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + kubernetes-etcd-peer: + certificates: + - document_name: kubernetes-etcd-genesis-peer + common_name: kubernetes-etcd-genesis-peer + hosts: + - n0 + - 192.168.77.10 + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - document_name: kubernetes-etcd-n0-peer + common_name: kubernetes-etcd-n0-peer + hosts: + - n0 + - 192.168.77.10 + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - document_name: kubernetes-etcd-n1-peer + common_name: kubernetes-etcd-n1-peer + hosts: + - n1 + - 192.168.77.11 + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + calico-etcd: + description: Certificates for Calico etcd client traffic + certificates: + - document_name: calico-etcd-anchor + description: anchor + common_name: anchor + - document_name: calico-etcd-n0 + common_name: calico-etcd-n0 + hosts: + - n0 + - 192.168.77.10 + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: calico-etcd-n1 + common_name: calico-etcd-n1 + hosts: + - n1 + - 192.168.77.11 + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: calico-node + common_name: calcico-node + calico-etcd-peer: + description: Certificates for Calico etcd clients + certificates: + - document_name: calico-etcd-n0-peer + common_name: calico-etcd-n0-peer + hosts: + - n0 + - 192.168.77.10 + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: calico-etcd-n1-peer + common_name: calico-etcd-n1-peer + hosts: + - n1 + - 192.168.77.11 + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: calico-node-peer + common_name: calcico-node-peer + keypairs: + - name: service-account + description: Service account signing key for use by Kubernetes controller-manager. +... diff --git a/tests/unit/builder_data/simple/armada-resources.yaml b/tests/unit/builder_data/simple/armada-resources.yaml new file mode 100644 index 00000000..ace41f9c --- /dev/null +++ b/tests/unit/builder_data/simple/armada-resources.yaml @@ -0,0 +1,1061 @@ +--- +schema: armada/Manifest/v1 +metadata: + schema: metadata/Document/v1 + name: cluster-bootstrap + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + release_prefix: ucp + chart_groups: + - kubernetes-proxy + - container-networking + - dns + - kubernetes + - ucp-services +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-proxy + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + description: Kubernetes proxy + sequenced: true + chart_group: + - kubernetes-proxy +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: container-networking + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + description: Container networking via Calico + sequenced: true + chart_group: + - calico-etcd + - calico +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: dns + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + description: Cluster DNS + chart_group: + - coredns +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + description: Kubernetes components + sequenced: true + chart_group: + - haproxy + - kubernetes-etcd + - kubernetes-apiserver + - kubernetes-controller-manager + - kubernetes-scheduler + - tiller +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: ucp-services + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + description: UCP platform components + sequenced: true + chart_group: + - promenade +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: helm-toolkit + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + chart_name: helm-toolkit + release: helm-toolkit + namespace: helm-toolkit + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: {} + source: + type: git + location: https://git.openstack.org/openstack/openstack-helm-infra + subpath: helm-toolkit + reference: master + dependencies: [] +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: helm-toolkit-pinned + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + chart_name: helm-toolkit + release: helm-toolkit + namespace: helm-toolkit + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: {} + source: + type: git + location: https://git.openstack.org/openstack/openstack-helm + subpath: helm-toolkit + reference: f902cd14fac7de4c4c9f7d019191268a6b4e9601 + dependencies: [] +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: infra-helm-toolkit + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + chart_name: infra-helm-toolkit + release: infra-helm-toolkit + namespace: infra-helm-toolkit + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: {} + source: + type: git + location: https://git.openstack.org/openstack/openstack-helm-infra + subpath: helm-toolkit + reference: master + dependencies: [] +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-proxy + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + chart_name: proxy + release: kubernetes-proxy + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: + images: + tags: + proxy: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + network: + kubernetes_netloc: 127.0.0.1:6553 + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: proxy + dependencies: + - helm-toolkit + +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: calico-etcd + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: calico-etcd + path: . + dest: + path: '.values.secrets.tls.client.ca' + - + src: + schema: deckhand/CertificateAuthority/v1 + name: calico-etcd-peer + path: . + dest: + path: '.values.secrets.tls.peer.ca' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-anchor + path: . + dest: + path: '.values.secrets.anchor.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-anchor + path: . + dest: + path: '.values.secrets.anchor.tls.key' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n0 + path: . + dest: + path: '.values.nodes[0].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n0 + path: . + dest: + path: '.values.nodes[0].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n0-peer + path: . + dest: + path: '.values.nodes[0].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n0-peer + path: . + dest: + path: '.values.nodes[0].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n1 + path: . + dest: + path: '.values.nodes[1].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n1 + path: . + dest: + path: '.values.nodes[1].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n1-peer + path: . + dest: + path: '.values.nodes[1].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n1-peer + path: . + dest: + path: '.values.nodes[1].tls.peer.key' + +data: + chart_name: etcd + release: calico-etcd + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: + anchor: + etcdctl_endpoint: 10.96.232.136 + labels: + anchor: + node_selector_key: calico-etcd + node_selector_value: enabled + secrets: + anchor: + tls: + cert: placeholder + key: placeholder + tls: + client: + ca: placeholder + peer: + ca: placeholder + etcd: + host_data_path: /var/lib/etcd/calico + host_etc_path: /etc/etcd/calico + bootstrapping: + enabled: true + host_directory: /var/lib/anchor + filename: calico-etcd-bootstrap + images: + tags: + etcd: quay.io/coreos/etcd:v3.2.14 + etcdctl: quay.io/coreos/etcd:v3.2.14 + nodes: + - name: n0 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + - name: n1 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + service: + name: calico-etcd + ip: 10.96.232.136 + network: + service_client: + name: service_client + port: 6666 + target_port: 6666 + service_peer: + name: service_peer + port: 6667 + target_port: 6667 + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: etcd + dependencies: + - helm-toolkit + +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: calico + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: calico-etcd + path: . + dest: + path: '.values.endpoints.etcd.auth.client.tls.ca' + - + src: + schema: deckhand/Certificate/v1 + name: calico-node + path: . + dest: + path: '.values.endpoints.etcd.auth.client.tls.crt' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-node + path: . + dest: + path: '.values.endpoints.etcd.auth.client.tls.key' + +data: + chart_name: calico + release: calico + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: + conf: + cni_network_config: + name: k8s-pod-network + cniVersion: 0.1.0 + type: calico + etcd_endpoints: __ETCD_ENDPOINTS__ + etcd_ca_cert_file: /etc/calico/pki/ca + etcd_cert_file: /etc/calico/pki/crt + etcd_key_file: /etc/calico/pki/key + log_level: debug + mtu: 1500 + ipam: + type: calico-ipam + policy: + type: k8s + k8s_api_root: https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__ + k8s_auth_token: __SERVICEACCOUNT_TOKEN__ + + policy_controller: + K8S_API: "https://10.96.0.1:443" + + node: + CALICO_STARTUP_LOGLEVEL: INFO + CLUSTER_TYPE: + - k8s + - bgp + IP_AUTODETECTION_METHOD: interface=ens3 + WAIT_FOR_STORAGE: "true" + + endpoints: + etcd: + hosts: + default: calico-etcd + host_fqdn_override: + default: 10.96.232.136 + scheme: + default: https + + networking: + podSubnet: 10.97.0.0/16 + mtu: 1500 + + images: + tags: + calico_node: quay.io/calico/node:v2.6.5 + calico_cni: quay.io/calico/cni:v1.11.2 + calico_ctl: quay.io/calico/ctl:v1.6.2 + calico_settings: quay.io/calico/ctl:v1.6.2 + calico_kube_policy_controller: quay.io/calico/kube-policy-controller:v0.7.0 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 + + manifests: + daemonset_calico_etcd: false + job_image_repo_sync: false + service_calico_etcd: false + source: + type: git + location: https://git.openstack.org/openstack/openstack-helm-infra + reference: master + subpath: calico + dependencies: + - infra-helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: coredns + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + chart_name: coredns + release: coredns + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + test: true + upgrade: + no_hooks: true + values: + conf: + test: + names_to_resolve: + - att.com + - calico-etcd.kube-system.svc.cluster.local + - google.com + - kubernetes-etcd.kube-system.svc.cluster.local + - kubernetes.default.svc.cluster.local + + images: + tags: + coredns: coredns/coredns:1.1.2 + test: coredns/coredns:1.1.2 + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: coredns + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: haproxy + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + chart_name: haproxy + release: haproxy + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + test: true + upgrade: + no_hooks: true + values: + conf: + anchor: + kubernetes_url: https://10.96.0.1:443 + services: + kube-system: + kubernetes-apiserver: + server_opts: "check port 6443" + conf_parts: + global: + - timeout connect 5000ms + - timeout client 30s + - timeout server 30s + frontend: + - mode tcp + - bind *:6553 + backend: + - mode tcp + - option tcp-check + - option redispatch + kubernetes-etcd: + server_opts: "check port 2379" + conf_parts: + frontend: + - mode tcp + - bind *:2378 + backend: + - mode tcp + - option tcp-check + - option redispatch + + images: + tags: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + haproxy: haproxy:1.8.3 + test: python:3.6 + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: haproxy + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-apiserver + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: . + dest: + path: .values.secrets.tls.ca + + - + src: + schema: deckhand/Certificate/v1 + name: apiserver + path: . + dest: + path: .values.secrets.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: apiserver + path: . + dest: + path: .values.secrets.tls.key + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes-etcd + path: . + dest: + path: .values.secrets.etcd.tls.ca + - + src: + schema: deckhand/Certificate/v1 + name: apiserver-etcd + path: . + dest: + path: .values.secrets.etcd.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: apiserver-etcd + path: . + dest: + path: .values.secrets.etcd.tls.key + - + src: + schema: deckhand/PublicKey/v1 + name: service-account + path: . + dest: + path: .values.secrets.service_account.public_key + +data: + chart_name: apiserver + release: kubernetes-apiserver + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: + command_prefix: + - /apiserver + - --authorization-mode=Node,RBAC + - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds + - --service-cluster-ip-range=10.96.0.0/16 + - --endpoint-reconciler-type=lease + apiserver: + etcd: + endpoints: https://127.0.0.1:2378 + images: + tags: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + apiserver: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + secrets: + service_account: + public_key: placeholder + tls: + ca: placeholder + cert: placeholder + key: placeholder + etcd: + tls: + ca: placeholder + cert: placeholder + key: placeholder + network: + kubernetes_service_ip: 10.96.0.1 + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: apiserver + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-controller-manager + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: . + dest: + path: .values.secrets.tls.ca + + - + src: + schema: deckhand/Certificate/v1 + name: controller-manager + path: . + dest: + path: .values.secrets.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: controller-manager + path: . + dest: + path: .values.secrets.tls.key + - + src: + schema: deckhand/PrivateKey/v1 + name: service-account + path: . + dest: + path: .values.secrets.service_account.private_key + +data: + chart_name: controller_manager + release: kubernetes-controller-manager + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: + images: + tags: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + controller_manager: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + secrets: + service_account: + private_key: placeholder + tls: + ca: placeholder + cert: placeholder + key: placeholder + network: + kubernetes_netloc: 127.0.0.1:6553 + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: controller_manager + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-scheduler + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: . + dest: + path: .values.secrets.tls.ca + - + src: + schema: deckhand/Certificate/v1 + name: scheduler + path: . + dest: + path: .values.secrets.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: scheduler + path: . + dest: + path: .values.secrets.tls.key + +data: + chart_name: scheduler + release: kubernetes-scheduler + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: + secrets: + tls: + ca: placeholder + cert: placeholder + key: placeholder + + network: + kubernetes_netloc: 127.0.0.1:6553 + + images: + tags: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + scheduler: gcr.io/google_containers/hyperkube-amd64:v1.10.2 + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: scheduler + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-etcd + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes-etcd + path: . + dest: + path: '.values.secrets.tls.client.ca' + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes-etcd-peer + path: . + dest: + path: '.values.secrets.tls.peer.ca' + + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-anchor + path: . + dest: + path: '.values.secrets.anchor.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-anchor + path: . + dest: + path: '.values.secrets.anchor.tls.key' + + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-n0 + path: . + dest: + path: '.values.nodes[0].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-n0 + path: . + dest: + path: '.values.nodes[0].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-n0-peer + path: . + dest: + path: '.values.nodes[0].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-n0-peer + path: . + dest: + path: '.values.nodes[0].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-n1 + path: . + dest: + path: '.values.nodes[1].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-n1 + path: . + dest: + path: '.values.nodes[1].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-n1-peer + path: . + dest: + path: '.values.nodes[1].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-n1-peer + path: . + dest: + path: '.values.nodes[1].tls.peer.key' + +data: + chart_name: etcd + release: kubernetes-etcd + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + test: true + upgrade: + no_hooks: true + values: + anchor: + etcdctl_endpoint: kubernetes-etcd.kube-system.svc.cluster.local + labels: + anchor: + node_selector_key: kubernetes-etcd + node_selector_value: enabled + secrets: + anchor: + tls: + cert: placeholder + key: placeholder + tls: + client: + ca: placeholder + peer: + ca: placeholder + etcd: + host_data_path: /var/lib/etcd/kubernetes + host_etc_path: /etc/etcd/kubernetes + images: + tags: + etcd: quay.io/coreos/etcd:v3.2.14 + etcdctl: quay.io/coreos/etcd:v3.2.14 + nodes: + - name: n0 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + - name: n1 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + service: + name: kubernetes-etcd + network: + service_client: + name: service_client + port: 2379 + target_port: 2379 + service_peer: + name: service_peer + port: 2380 + target_port: 2380 + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: etcd + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: tiller + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + chart_name: tiller + release: tiller + namespace: kube-system + install: + no_hooks: false + upgrade: + no_hooks: false + timeout: 600 + wait: + timeout: 600 + values: + images: + tags: + tiller: gcr.io/kubernetes-helm/tiller:v2.9.1 + labels: + node_selector_key: ucp-control-plane + node_selector_value: enabled + source: + type: git + location: https://github.com/openstack/airship-armada + subpath: charts/tiller + reference: 8d1521e96c6b3163f7f6598ef15a11af0052cf04 + dependencies: + - helm-toolkit-pinned +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: promenade + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + chart_name: promenade + release: promenade + namespace: ucp + timeout: 600 + wait: + timeout: 600 + test: true + values: + pod: + env: + promenade_api: + - name: PROMENADE_DEBUG + value: '1' + conf: + paste: + app:promenade-api: + disable: keystone + pipeline:main: + pipeline: noauth promenade-api + images: + tags: + promenade: quay.io/airshipit/promenade:master + manifests: + job_ks_endpoints: false + job_ks_service: false + job_ks_user: false + secret_keystone: false + upgrade: + no_hooks: true + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: promenade + dependencies: + - helm-toolkit +... diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py new file mode 100644 index 00000000..1cb6b876 --- /dev/null +++ b/tests/unit/test_builder.py @@ -0,0 +1,57 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from promenade import builder, generator, config, encryption_method +import copy +import os +import pytest + + +def load_full_config(dirname): + this_dir = os.path.dirname(os.path.realpath(__file__)) + search_dir = os.path.join(this_dir, 'builder_data', dirname) + streams = [] + for filename in os.listdir(search_dir): + stream = open(os.path.join(search_dir, filename)) + streams.append(stream) + + raw_config = config.Configuration.from_streams( + allow_missing_substitutions=True, + debug=True, + streams=streams, + substitute=True, + validate=False, + ) + g = generator.Generator(raw_config, block_strings=False) + g.generate() + + documents = copy.deepcopy(raw_config.documents) + documents.extend(copy.deepcopy(g.get_documents())) + + return config.Configuration( + allow_missing_substitutions=False, + debug=True, + documents=documents, + substitute=True, + validate=True, + ) + + +def test_build_simple(): + b = builder.Builder(load_full_config('simple')) + genesis_script = b.build_genesis_script() + assert len(genesis_script) > 0 + + n1_join_script = b.build_node_script('n1') + assert len(n1_join_script) > 0 diff --git a/tools/g2/lib/config.sh b/tools/g2/lib/config.sh index d5013b9b..ed263100 100644 --- a/tools/g2/lib/config.sh +++ b/tools/g2/lib/config.sh @@ -6,6 +6,7 @@ export NGINX_DIR="${TEMP_DIR}/nginx" export NGINX_URL="http://192.168.77.1:7777" export PROMENADE_BASE_URL="http://promenade-api.ucp.svc.cluster.local" export PROMENADE_DEBUG=${PROMENADE_DEBUG:-0} +export PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY:-testkey} export REGISTRY_DATA_DIR=${REGISTRY_DATA_DIR:-/mnt/registry} export VIRSH_POOL=${VIRSH_POOL:-promenade} export VIRSH_POOL_PATH=${VIRSH_POOL_PATH:-/var/lib/libvirt/promenade} diff --git a/tools/g2/stages/build-scripts.sh b/tools/g2/stages/build-scripts.sh index 77796940..22fa3920 100755 --- a/tools/g2/stages/build-scripts.sh +++ b/tools/g2/stages/build-scripts.sh @@ -13,6 +13,7 @@ docker run --rm -t \ -w /target \ -v "${TEMP_DIR}:/target" \ -e "PROMENADE_DEBUG=${PROMENADE_DEBUG}" \ + -e "PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY}" \ "${IMAGE_PROMENADE}" \ promenade \ build-all \ diff --git a/tools/g2/stages/genesis.sh b/tools/g2/stages/genesis.sh index 61461327..2f762386 100755 --- a/tools/g2/stages/genesis.sh +++ b/tools/g2/stages/genesis.sh @@ -7,7 +7,7 @@ source "${GATE_UTILS}" rsync_cmd "${TEMP_DIR}/scripts"/*genesis* "${GENESIS_NAME}:/root/promenade/" set -o pipefail -ssh_cmd "${GENESIS_NAME}" /root/promenade/genesis.sh 2>&1 | tee -a "${LOG_FILE}" +ssh_cmd "${GENESIS_NAME}" env "PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY}" /root/promenade/genesis.sh 2>&1 | tee -a "${LOG_FILE}" ssh_cmd "${GENESIS_NAME}" /root/promenade/validate-genesis.sh 2>&1 | tee -a "${LOG_FILE}" set +o pipefail diff --git a/tools/install-external-deps.sh b/tools/install-external-deps.sh new file mode 100755 index 00000000..053ad953 --- /dev/null +++ b/tools/install-external-deps.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Installs external dependencies required for basic testing + +set -ex + +CFSSL_URL=${CFSSL_URL:-https://pkg.cfssl.org/R1.2/cfssl_linux-amd64} + +if [[ ! $(which cfssl) ]]; then + TMP_DIR=$(mktemp -d) + pushd "${TMP_DIR}" + curl -Lo cfssl "${CFSSL_URL}" + chmod 755 cfssl + sudo mv cfssl /usr/local/bin/ + popd + rm -rf "${TMP_DIR}" +fi diff --git a/tools/lint_gate.sh b/tools/lint_gate.sh index 9bb1a7cb..1abef0e6 100755 --- a/tools/lint_gate.sh +++ b/tools/lint_gate.sh @@ -12,7 +12,7 @@ done if [[ -x $(which shellcheck) ]]; then echo Checking shell scripts.. - shellcheck -s bash -e SC2029 "${WORKSPACE}"/tools/cleanup.sh "${WORKSPACE}"/tools/*gate*.sh "${WORKSPACE}"/tools/g2/stages/* "${WORKSPACE}"/tools/g2/lib/* + shellcheck -s bash -e SC2029 "${WORKSPACE}"/tools/cleanup.sh "${WORKSPACE}"/tools/*gate*.sh "${WORKSPACE}"/tools/g2/stages/* "${WORKSPACE}"/tools/g2/lib/* "${WORKSPACE}"/tools/install-external-deps.sh else echo No shellcheck executable found. Please, install it. exit 1