From 9b165b6c7002002e66c4a53af4f928320ca38fe7 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Mon, 19 Jun 2017 09:27:18 -0500 Subject: [PATCH] implment initial config + pki generation --- promenade/cli.py | 16 +- promenade/config.py | 156 +++++++++----------- promenade/generator.py | 327 +++++++++++++++++++++++++++++++++++++++++ promenade/pki.py | 149 ++++++++++++++++++- 4 files changed, 558 insertions(+), 90 deletions(-) create mode 100644 promenade/generator.py diff --git a/promenade/cli.py b/promenade/cli.py index cf631cb0..6aad49ac 100644 --- a/promenade/cli.py +++ b/promenade/cli.py @@ -1,4 +1,4 @@ -from . import logging, operator +from . import generator, logging, operator import click __all__ = [] @@ -57,3 +57,17 @@ def join(*, asset_dir, config_path, hostname, target_dir): target_dir=target_dir) op.join(asset_dir=asset_dir) + + +@promenade.command(help='Generate certs and keys') +@click.option('-c', '--config-path', type=click.File(), + required=True, + help='Location of cluster configuration data.') +@click.option('-o', '--output-dir', default='.', + type=click.Path(exists=True, file_okay=False, dir_okay=True, + resolve_path=True), + required=True, + help='Location to write complete cluster configuration.') +def generate(*, config_path, output_dir): + g = generator.Generator.from_config(config_path=config_path) + g.generate_all(output_dir) diff --git a/promenade/config.py b/promenade/config.py index 57881012..8a782f33 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -1,113 +1,95 @@ from . import logging -from operator import itemgetter +from operator import attrgetter, itemgetter import itertools import yaml -__all__ = ['load_config_file'] +__all__ = ['Configuration', 'Document', 'load'] LOG = logging.getLogger(__name__) -def load_config_file(*, config_path, hostname): - LOG.debug('Loading genesis configuration from "%s"', config_path) - cluster_data = yaml.load(open(config_path)) - LOG.debug('Loaded genesis configruation from "%s"', config_path) - node_data = extract_node_data(hostname, cluster_data) +def load(f): + return Configuration(list(map(Document, yaml.load_all(f)))) - return { - 'cluster_data': cluster_data, - 'node_data': node_data, + +class Document: + KEYS = { + 'apiVersion', + 'metadata', + 'kind', + 'spec', } - -def extract_node_data(hostname, cluster_data): - genesis = _extract_genesis_data(cluster_data['nodes']) - masters = _extract_master_data(cluster_data['nodes']) - return { - 'cluster': cluster_data['nodes'], - 'current_node': _extract_current_node_data(cluster_data['nodes'], - hostname), - 'etcd': _extract_etcd_data(hostname, genesis, masters), - 'genesis': genesis, - 'masters': masters, - 'network': cluster_data['network'], + SUPPORTED_KINDS = { + 'Certificate', + 'CertificateAuthority', + 'CertificateAuthorityKey', + 'CertificateKey', + 'Cluster', + 'Etcd', + 'Masters', + 'Network', + 'Node', + 'PrivateKey', + 'PublicKey', } + def __init__(self, data): + assert set(data.keys()) == self.KEYS + assert data['apiVersion'] == 'promenade/v1' + assert data['kind'] in self.SUPPORTED_KINDS -def _extract_etcd_data(hostname, genesis, masters): - LOG.info('hostname=%r genesis=%r masters=%r', - hostname, genesis, masters) - non_genesis_masters = [d for d in masters if d['hostname'] != genesis['hostname']] - boot_order = [genesis] + sorted(non_genesis_masters, key=itemgetter('hostname')) + self.data = data - result = { - 'boot_order': boot_order, - 'env': {}, - } + @property + def kind(self): + return self.data['kind'] - peers = [ - { - 'hostname': 'auxiliary-etcd-%d' % i, - 'peer_port': 2380 + (i + 1) * 10000 - } - for i in range(2) - ] - peers.append({ - 'hostname': genesis['hostname'], - }) + @property + def target(self): + return self.metadata.get('target') - if hostname == genesis['hostname']: - result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'new' - else: - result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'existing' - for host in non_genesis_masters: - peers.append({'hostname': host['hostname']}) + @property + def metadata(self): + return self.data['metadata'] - result['env']['ETCD_INITIAL_CLUSTER'] = ','.join( - '%s=https://%s:%d' % (p['hostname'], p['hostname'], p.get('peer_port', 2380)) - for p in peers) - - return result + def __getitem__(self, key): + return self.data['spec'][key] -def _extract_current_node_data(nodes, hostname): - base = nodes[hostname] - return { - 'hostname': hostname, - 'labels': _extract_node_labels(base), - **base, - } +class Configuration: + def __init__(self, documents): + self.documents = sorted(documents, key=attrgetter('kind', 'target')) + def __getitem__(self, key): + results = [d for d in self.documents if d.kind == key] + if len(results) < 1: + raise KeyError + elif len(results) > 1: + raise KeyError('Too many results.') + else: + return results[0] -ROLE_LABELS = { - 'genesis': [ - 'promenade=genesis', - ], - 'master': [ - 'node-role.kubernetes.io/master=', - ], -} + def iterate(self, *, kind=None, target=None): + if target: + docs = self._iterate_with_target(target) + else: + docs = self.documents + for document in docs: + if not kind or document.kind == kind: + yield document -def _extract_node_labels(data): - labels = set(itertools.chain.from_iterable( - map(lambda k: ROLE_LABELS.get(k, []), ['common'] + data['roles']))) - labels.update(data.get('additional_labels', [])) - return sorted(labels) + def _iterate_with_target(self, target): + for document in self.documents: + if document.target == target or document.target == 'all': + yield document - -def _extract_genesis_data(nodes): - for hostname, node in nodes.items(): - if 'genesis' in node['roles']: - return { - 'hostname': hostname, - 'ip': node['ip'], - } - - -def _extract_master_data(nodes): - return sorted(({'hostname': hostname, 'ip': node['ip']} - for hostname, node in nodes.items() - if 'master' in node['roles']), - key=itemgetter('hostname')) + def write(self, path): + with open(path, 'w') as f: + yaml.dump_all(map(attrgetter('data'), self.documents), + default_flow_style=False, + explicit_start=True, + indent=2, + stream=f) diff --git a/promenade/generator.py b/promenade/generator.py new file mode 100644 index 00000000..90b3fb81 --- /dev/null +++ b/promenade/generator.py @@ -0,0 +1,327 @@ +from . import config, logging, pki +import os + +__all__ = ['Generator'] + + +LOG = logging.getLogger(__name__) + + +class Generator: + @classmethod + def from_config(cls, *, config_path): + return cls(input_config=(config.load(config_path))) + + def __init__(self, *, input_config): + self.input_config = input_config + + self.validate() + + def validate(self): + required_kinds = ['Cluster', 'Network'] + for required_kind in required_kinds: + try: + self.input_config[required_kind] + except KeyError: + LOG.error('Generator requires one "%s" document to function.', + required_kind) + raise + + assert self.input_config['Cluster'].metadata['name'] \ + == self.input_config['Network'].metadata['cluster'] + + def generate_all(self, output_dir): + cluster = self.input_config['Cluster'] + network = self.input_config['Network'] + + cluster_name = cluster.metadata['name'] + LOG.info('Generating configuration for cluster "%s"', cluster_name) + masters = self.construct_masters(cluster_name) + + LOG.info('Generating common PKI for cluster "%s"', cluster_name) + keys = pki.PKI(cluster_name) + cluster_ca, cluster_ca_key = keys.generate_ca( + ca_name='cluster', + cert_target='all', + key_target='masters') + etcd_client_ca, etcd_client_ca_key = keys.generate_ca( + ca_name='etcd-client', + cert_target='all', + key_target='masters') + etcd_peer_ca, etcd_peer_ca_key = keys.generate_ca( + ca_name='etcd-peer', + cert_target='all', + key_target='masters') + + admin_cert, admin_cert_key = keys.generate_certificate( + name='admin', + ca_name='cluster', + groups=['system:masters'], + target='masters', + ) + + sa_pub, sa_priv = keys.generate_keypair( + name='service-account', + target='masters', + ) + + config.Configuration([ + cluster_ca, + cluster_ca_key, + etcd_client_ca, + etcd_client_ca_key, + etcd_peer_ca, + etcd_peer_ca_key, + sa_pub, + sa_priv, + ]).write(os.path.join(output_dir, 'admin-bundle.yaml')) + + for hostname, data in cluster['nodes'].items(): + if 'genesis' in data['roles']: + genesis_hostname = hostname + break + + for hostname, data in cluster['nodes'].items(): + LOG.debug('Generating configuration & PKI for hostname=%s', + hostname) + node = _construct_node_config(cluster_name, hostname, data) + + kubelet_cert, kubelet_cert_key = keys.generate_certificate( + alias='kubelet', + name='system:node:%s' % hostname, + ca_name='cluster', + groups=['system:nodes'], + hosts=[ + hostname, + data['ip'], + ], + target=hostname) + + proxy_cert, proxy_cert_key = keys.generate_certificate( + alias='proxy', + name='system:kube-proxy', + ca_name='cluster', + hosts=[ + hostname, + data['ip'], + ], + target=hostname) + + common_documents = [ + cluster_ca, + kubelet_cert, + kubelet_cert_key, + masters, + network, + node, + proxy_cert, + proxy_cert_key, + ] + role_specific_documents = [] + + if 'master' in data['roles']: + role_specific_documents.extend([ + admin_cert, + admin_cert_key, + etcd_client_ca, + etcd_peer_ca, + sa_priv, + sa_pub, + ]) + if 'genesis' not in data['roles']: + role_specific_documents.append( + _master_etcd_config(cluster_name, genesis_hostname, + hostname, masters) + ) + role_specific_documents.extend(_master_config(hostname, data, + masters, network, keys)) + + if 'genesis' in data['roles']: + role_specific_documents.extend(_genesis_config(hostname, data, + masters, network, keys)) + role_specific_documents.append(_genesis_etcd_config(cluster_name, hostname)) + node.data['is_genesis'] = True + + c = config.Configuration(common_documents + role_specific_documents) + c.write(os.path.join(output_dir, hostname + '.yaml')) + + def construct_masters(self, cluster_name): + masters = [] + for hostname, data in self.input_config['Cluster']['nodes'].items(): + if 'master' in data['roles'] or 'genesis' in data['roles']: + masters.append({'hostname': hostname, 'ip': data['ip']}) + + return config.Document({ + 'apiVersion': 'promenade/v1', + 'kind': 'Masters', + 'metadata': { + 'cluster': cluster_name, + 'target': 'all', + }, + 'spec': { + 'nodes': masters, + }, + }) + + +def _master_etcd_config(cluster_name, genesis_hostname, hostname, masters): + initial_cluster = ['%s=https://%s:2380' % (m['hostname'], + m['hostname']) + for m in masters['nodes']] + initial_cluster.extend([ + 'auxiliary-etcd-0=https://%s:12380' % genesis_hostname, + 'auxiliary-etcd-1=https://%s:22380' % genesis_hostname, + ]) + return _etcd_config(cluster_name, target=hostname, + initial_cluster=initial_cluster, + initial_cluster_state='existing') + + +def _genesis_etcd_config(cluster_name, hostname): + initial_cluster = [ + '%s=https://%s:2380' % (hostname, hostname), + 'auxiliary-etcd-0=https://%s:12380' % hostname, + 'auxiliary-etcd-1=https://%s:22380' % hostname, + ] + return _etcd_config(cluster_name, target=hostname, + initial_cluster=initial_cluster, + initial_cluster_state='new') + + +def _etcd_config(cluster_name, *, target, + initial_cluster, initial_cluster_state): + return config.Document({ + 'apiVersion': 'promenade/v1', + 'kind': 'Etcd', + 'metadata': { + 'cluster': cluster_name, + 'target': target, + }, + 'spec': { + 'initial_cluster': initial_cluster, + 'initial_cluster_state': initial_cluster_state, + }, + }) + + +def _master_config(hostname, host_data, masters, network, keys): + kube_domains = [ + 'kubernetes', + 'kubernetes.default', + 'kubernetes.default.svc', + 'kubernetes.default.svc.cluster.local', + '127.0.0.1', + ] + docs = [] + + docs.extend(keys.generate_certificate( + alias='etcd-client', + name='etcd:client:%s' % hostname, + ca_name='etcd-client', + hosts=kube_domains + [hostname, host_data['ip']], + target=hostname, + )) + docs.extend(keys.generate_certificate( + alias='etcd-peer', + name='etcd:peer:%s' % hostname, + ca_name='etcd-peer', + hosts=kube_domains + [hostname, host_data['ip']], + target=hostname, + )) + + docs.extend(keys.generate_certificate( + alias='apiserver', + name='apiserver:%s' % hostname, + ca_name='cluster', + hosts=kube_domains + [ + network['kube_service_ip'], + hostname, + host_data['ip'], + ], + target=hostname, + )) + + docs.extend(keys.generate_certificate( + alias='controller-manager', + name='system:kube-controller-manager', + ca_name='cluster', + hosts=[ + hostname, + host_data['ip'], + ], + target=hostname, + )) + + docs.extend(keys.generate_certificate( + alias='scheduler', + name='system:kube-scheduler', + ca_name='cluster', + hosts=[ + hostname, + host_data['ip'], + ], + target=hostname, + )) + + return docs + + +def _genesis_config(hostname, host_data, masters, network, keys): + docs = [] + + for i in range(2): + docs.extend(keys.generate_certificate( + name='auxiliary-etcd-client-%d' % i, + ca_name='etcd-client', + hosts=[hostname, host_data['ip']], + target=hostname, + )) + docs.extend(keys.generate_certificate( + name='auxiliary-etcd-client-%d' % i, + ca_name='etcd-peer', + hosts=[hostname, host_data['ip']], + target=hostname, + )) + + return docs + + +def _construct_node_config(cluster_name, hostname, data): + spec = { + 'hostname': hostname, + 'ip': data['ip'], + 'labels': _labels(data['roles'], data.get('additional_labels', [])), + 'templates': _templates(data['roles']), + } + + return config.Document({ + 'apiVersion': 'promenade/v1', + 'kind': 'Node', + 'metadata': { + 'cluster': cluster_name, + 'target': hostname, + }, + 'spec': spec, + }) + + +ROLE_LABELS = { + 'genesis': [ + 'promenade=genesis', + ], + 'master': [ + 'node-role.kubernetes.io/master=', + ], +} + + +def _labels(roles, additional_labels): + result = set() + for role in roles: + result.update(ROLE_LABELS.get(role, [])) + result.update(additional_labels) + return sorted(result) + + +def _templates(roles): + return ['common'] + roles diff --git a/promenade/pki.py b/promenade/pki.py index 305f4275..7863efb6 100644 --- a/promenade/pki.py +++ b/promenade/pki.py @@ -1,15 +1,160 @@ -from promenade import logging +from . import config, logging +import json import os import shutil import subprocess import tempfile +import yaml -__all__ = ['generate_keys'] +__all__ = ['PKI'] LOG = logging.getLogger(__name__) +class PKI: + def __init__(self, cluster_name, *, ca_config=None): + self.certificate_authorities = {} + self.cluster_name = cluster_name + + self._ca_config_string = None + if ca_config: + self._ca_config_string = json.dumps(ca_config) + + @property + def ca_config(self): + if not self._ca_config_string: + self._ca_config_string = json.dumps({ + 'signing': { + 'default': { + 'expiry': '8760h', + 'usages': ['signing', 'key encipherment', 'server auth', 'client auth'], + }, + }, + }) + return self._ca_config_string + + def generate_ca(self, *, ca_name, cert_target, key_target): + result = self._cfssl(['gencert', '-initca', 'csr.json'], + files={ + 'csr.json': self.csr( + name='Kubernetes', + groups=['Kubernetes']), + }) + LOG.debug('ca_cert=%r', result['cert']) + self.certificate_authorities[ca_name] = result + + return (self._wrap('CertificateAuthority', result['cert'], + name=ca_name, + target=cert_target), + self._wrap('CertificateAuthorityKey', result['key'], + name=ca_name, + target=key_target)) + + def generate_keypair(self, *, alias=None, name, target): + priv_result = self._openssl(['genrsa', '-out', 'priv.pem']) + pub_result = self._openssl(['rsa', '-in', 'priv.pem', '-pubout', '-out', 'pub.pem'], + files={ + 'priv.pem': priv_result['priv.pem'], + }) + + if not alias: + alias = name + + return (self._wrap('PublicKey', pub_result['pub.pem'], + name=alias, + target=target), + self._wrap('PrivateKey', priv_result['priv.pem'], + name=alias, + target=target)) + + + def generate_certificate(self, *, alias=None, ca_name, groups=[], hosts=[], name, target): + result = self._cfssl( + ['gencert', + '-ca', 'ca.pem', + '-ca-key', 'ca-key.pem', + '-config', 'ca-config.json', + 'csr.json'], + files={ + 'ca-config.json': self.ca_config, + 'ca.pem': self.certificate_authorities[ca_name]['cert'], + 'ca-key.pem': self.certificate_authorities[ca_name]['key'], + 'csr.json': self.csr(name=name, groups=groups, hosts=hosts), + }) + + if not alias: + alias = name + + return (self._wrap('Certificate', result['cert'], + name=alias, + target=target), + self._wrap('CertificateKey', result['key'], + name=alias, + target=target)) + + def csr(self, *, name, groups=[], hosts=[], key={'algo': 'rsa', 'size': 2048}): + return json.dumps({ + 'CN': name, + 'key': key, + 'hosts': hosts, + 'names': [{'O': g} for g in groups], + }) + + def _cfssl(self, command, *, files=None): + if not files: + files = {} + with tempfile.TemporaryDirectory() as tmp: + for filename, data in files.items(): + with open(os.path.join(tmp, filename), 'w') as f: + f.write(data) + + return json.loads(subprocess.check_output( + ['cfssl'] + command, cwd=tmp)) + + def _openssl(self, command, *, files=None): + if not files: + files = {} + + with tempfile.TemporaryDirectory() as tmp: + for filename, data in files.items(): + with open(os.path.join(tmp, filename), 'w') as f: + f.write(data) + + subprocess.check_call(['openssl'] + command, cwd=tmp) + + result = {} + for filename in os.listdir(tmp): + if filename not in files: + with open(os.path.join(tmp, filename)) as f: + result[filename] = f.read() + + return result + + def _wrap(self, kind, data, **metadata): + return config.Document({ + 'apiVersion': 'promenade/v1', + 'kind': kind, + 'metadata': { + 'cluster': self.cluster_name, + **metadata, + }, + 'spec': { + 'data': block_literal(data), + }, + }) + + +class block_literal(str): pass + + +def block_literal_representer(dumper, data): + return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|') + + +yaml.add_representer(block_literal, block_literal_representer) + + CA_ONLY_MAP = { 'cluster-ca': [ 'kubelet',