From 866412ce627c7ad2502366a7aadaf917f8fae3ea Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Fri, 16 Jun 2017 14:13:09 -0500 Subject: [PATCH] use auxiliary etcd instances to smooth join process --- promenade/config.py | 25 ++- promenade/etcd.py | 24 --- promenade/kube.py | 27 --- promenade/operator.py | 15 +- .../common/etc/dnsmasq.d/auxilliary-etcd | 2 + .../kubernetes/cfssl/csr-configs/etcd.json | 19 ++ .../kubelet/manifests/auxilliary-etcd.yaml | 183 ++++++++++++++++++ 7 files changed, 221 insertions(+), 74 deletions(-) delete mode 100644 promenade/etcd.py delete mode 100644 promenade/kube.py create mode 100644 promenade/templates/common/etc/dnsmasq.d/auxilliary-etcd create mode 100644 promenade/templates/genesis/etc/kubernetes/cfssl/csr-configs/etcd.json create mode 100644 promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxilliary-etcd.yaml diff --git a/promenade/config.py b/promenade/config.py index 9536cacb..57881012 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -46,20 +46,27 @@ def _extract_etcd_data(hostname, genesis, masters): 'env': {}, } - peers = [] - for host in boot_order: - peers.append(host) - if host['hostname'] == hostname: - break - - result['env']['ETCD_INITIAL_CLUSTER'] = ','.join( - '%s=https://%s:2380' % (p['hostname'], p['hostname']) - for p in peers) + peers = [ + { + 'hostname': 'auxiliary-etcd-%d' % i, + 'peer_port': 2380 + (i + 1) * 10000 + } + for i in range(2) + ] + peers.append({ + 'hostname': genesis['hostname'], + }) if hostname == genesis['hostname']: result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'new' else: result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'existing' + for host in non_genesis_masters: + peers.append({'hostname': host['hostname']}) + + result['env']['ETCD_INITIAL_CLUSTER'] = ','.join( + '%s=https://%s:%d' % (p['hostname'], p['hostname'], p.get('peer_port', 2380)) + for p in peers) return result diff --git a/promenade/etcd.py b/promenade/etcd.py deleted file mode 100644 index 0f5302ab..00000000 --- a/promenade/etcd.py +++ /dev/null @@ -1,24 +0,0 @@ -from . import kube, logging - -__all__ = ['add_member'] - - -LOG = logging.getLogger(__name__) - - -def add_member(exec_pod, hostname, port): - opts = ' '.join([ - '--cacert', - '/etc/etcd-pki/cluster-ca.pem', - '--cert', - '/etc/etcd-pki/etcd.pem', - '--key', - '/etc/etcd-pki/etcd-key.pem', - ]) - result = kube.kc('exec', '-n', 'kube-system', '-t', exec_pod, '--', 'sh', '-c', - 'ETCDCTL_API=3 etcdctl %s member add %s --peer-urls https://%s:%d' - % (opts, hostname, hostname, port)) - if result.returncode != 0: - LOG.error('Failed to add etcd member. STDOUT: %r', result.stdout) - LOG.error('Failed to add etcd member. STDERR: %r', result.stderr) - result.check_returncode() diff --git a/promenade/kube.py b/promenade/kube.py deleted file mode 100644 index 59e1d04b..00000000 --- a/promenade/kube.py +++ /dev/null @@ -1,27 +0,0 @@ -from . import logging -import subprocess -import time - -__all__ = ['kc', 'wait_for_node'] - - -LOG = logging.getLogger(__name__) - - -def wait_for_node(node): - repeat = True - while repeat: - result = kc('get', 'nodes', node, '-o', - r'jsonpath={.status.conditions[?(@.type=="Ready")].status}') - if result.stdout == b'True': - repeat = False - else: - LOG.debug('Node "%s" not ready, waiting. stdout=%r stderr=%r', - node, result.stdout, result.stderr) - time.sleep(5) - - -def kc(*args): - return subprocess.run(['/target/usr/local/bin/kubectl', - '--kubeconfig', '/target/etc/kubernetes/genesis/kubeconfig.yaml', *args], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/promenade/operator.py b/promenade/operator.py index a239956a..b20b1c97 100644 --- a/promenade/operator.py +++ b/promenade/operator.py @@ -1,4 +1,4 @@ -from . import config, etcd, logging, kube, pki, renderer +from . import config, logging, pki, renderer import os import subprocess @@ -26,7 +26,6 @@ class Operator: def genesis(self, *, asset_dir=None): self.setup(asset_dir=asset_dir) - self.expand_etcd_cluster() def join(self, *, asset_dir=None): self.setup(asset_dir=asset_dir) @@ -63,15 +62,3 @@ class Operator: self.target_dir, '/bin/bash', '/usr/local/bin/bootstrap'], check=True) - - def expand_etcd_cluster(self): - for node in self.node_data['etcd']['boot_order'][1:]: - LOG.info('Waiting for Node "%s" to be Ready', node['hostname']) - kube.wait_for_node(node['hostname']) - LOG.info('Node "%s" Ready. Adding to etcd cluster.', node['hostname']) - etcd.add_member(self.genesis_etcd_pod, node['hostname'], port=2380) - LOG.info('Finished expanding etcd cluster.') - - @property - def genesis_etcd_pod(self): - return 'kube-etcd-%s' % self.node_data['genesis']['hostname'] diff --git a/promenade/templates/common/etc/dnsmasq.d/auxilliary-etcd b/promenade/templates/common/etc/dnsmasq.d/auxilliary-etcd new file mode 100644 index 00000000..9e51b8e0 --- /dev/null +++ b/promenade/templates/common/etc/dnsmasq.d/auxilliary-etcd @@ -0,0 +1,2 @@ +host-record=auxiliary-etcd-0,{{ genesis['ip'] }} +host-record=auxiliary-etcd-1,{{ genesis['ip'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/cfssl/csr-configs/etcd.json b/promenade/templates/genesis/etc/kubernetes/cfssl/csr-configs/etcd.json new file mode 100644 index 00000000..1dc33988 --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/cfssl/csr-configs/etcd.json @@ -0,0 +1,19 @@ +{ + "CN": "etcd:{{ current_node['hostname'] }}", + "hosts": [ + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + "127.0.0.1", + "{{ current_node['hostname'] }}", + "auxiliary-etcd-0", + "auxiliary-etcd-1", + "{{ current_node['ip'] }}", + "{{ network.kube_service_ip }}" + ], + "key": { + "algo": "rsa", + "size": 2048 + } +} diff --git a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxilliary-etcd.yaml b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxilliary-etcd.yaml new file mode 100644 index 00000000..dde18304 --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxilliary-etcd.yaml @@ -0,0 +1,183 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: auxiliary-etcd + namespace: kube-system + labels: + component: auxiliary-etcd + promenade: genesis +spec: + hostNetwork: true + containers: + - name: auxiliary-etcd-0 + image: quay.io/coreos/etcd:v3.0.17 + env: + - name: ETCD_NAME + value: auxiliary-etcd-0 + - name: ETCD_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_PEER_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_DATA_DIR + value: /var/lib/auxiliary-etcd-0 + - name: ETCD_TRUSTED_CA_FILE + value: /etc/etcd-pki/cluster-ca.pem + - name: ETCD_CERT_FILE + value: /etc/etcd-pki/etcd.pem + - name: ETCD_KEY_FILE + value: /etc/etcd-pki/etcd-key.pem + - name: ETCD_PEER_TRUSTED_CA_FILE + value: /etc/etcd-pki/cluster-ca.pem + - name: ETCD_PEER_CERT_FILE + value: /etc/etcd-pki/etcd.pem + - name: ETCD_PEER_KEY_FILE + value: /etc/etcd-pki/etcd-key.pem + - name: ETCD_ADVERTISE_CLIENT_URLS + value: https://$(ETCD_NAME):12379 + - name: ETCD_INITIAL_ADVERTISE_PEER_URLS + value: https://$(ETCD_NAME):12380 + - name: ETCD_INITIAL_CLUSTER_TOKEN + value: promenade-kube-etcd-token + - name: ETCD_LISTEN_CLIENT_URLS + value: https://0.0.0.0:12379 + - name: ETCD_LISTEN_PEER_URLS + value: https://0.0.0.0:12380 +{%- for env_name, env_value in etcd['env'].items() %} + - name: {{ env_name }} + value: {{ env_value }} +{%- endfor %} + ports: + - name: client + containerPort: 12379 + - name: peer + containerPort: 12380 + resources: + limits: + cpu: 100m + requests: + cpu: 100m + volumeMounts: + - name: data-0 + mountPath: /var/lib/auxiliary-etcd-0 + - name: pki + mountPath: /etc/etcd-pki + readOnly: true + - name: auxiliary-etcd-1 + image: quay.io/coreos/etcd:v3.0.17 + env: + - name: ETCD_NAME + value: auxiliary-etcd-1 + - name: ETCD_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_PEER_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_DATA_DIR + value: /var/lib/auxiliary-etcd-1 + - name: ETCD_TRUSTED_CA_FILE + value: /etc/etcd-pki/cluster-ca.pem + - name: ETCD_CERT_FILE + value: /etc/etcd-pki/etcd.pem + - name: ETCD_KEY_FILE + value: /etc/etcd-pki/etcd-key.pem + - name: ETCD_PEER_TRUSTED_CA_FILE + value: /etc/etcd-pki/cluster-ca.pem + - name: ETCD_PEER_CERT_FILE + value: /etc/etcd-pki/etcd.pem + - name: ETCD_PEER_KEY_FILE + value: /etc/etcd-pki/etcd-key.pem + - name: ETCD_ADVERTISE_CLIENT_URLS + value: https://$(ETCD_NAME):22379 + - name: ETCD_INITIAL_ADVERTISE_PEER_URLS + value: https://$(ETCD_NAME):22380 + - name: ETCD_INITIAL_CLUSTER_TOKEN + value: promenade-kube-etcd-token + - name: ETCD_LISTEN_CLIENT_URLS + value: https://0.0.0.0:22379 + - name: ETCD_LISTEN_PEER_URLS + value: https://0.0.0.0:22380 +{%- for env_name, env_value in etcd['env'].items() %} + - name: {{ env_name }} + value: {{ env_value }} +{%- endfor %} + ports: + - name: client + containerPort: 22379 + - name: peer + containerPort: 22380 + resources: + limits: + cpu: 100m + requests: + cpu: 100m + volumeMounts: + - name: data-1 + mountPath: /var/lib/auxiliary-etcd-1 + - name: pki + mountPath: /etc/etcd-pki + readOnly: true + - name: cluster-monitor + image: quay.io/coreos/etcd:v3.0.17 + command: + - sh + - -c + - |- + set -x + while true; do + if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -ge {{ masters | length }} ]; then + {%- for master in masters %} + etcdctl member add {{ master['hostname'] }} --peer-urls https://{{ master['hostname'] }}:2380 + {%- endfor %} + break + fi + done + while true; do + sleep 5 + if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -eq {{ 2 + (masters | length) }} ]; then + etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-1 | cut -d , -f 1) + etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-0 | cut -d , -f 1) + sleep 60 + rm -rf /var/lib/auxiliary-etcd-0 /var/lib/auxiliary-etcd-1 /etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml + sleep 10000 + fi + done + resources: + limits: + cpu: 100m + requests: + cpu: 100m + env: + - name: ETCDCTL_API + value: "3" + - name: ETCDCTL_CACERT + value: /etc/etcd-pki/cluster-ca.pem + - name: ETCDCTL_CERT + value: /etc/etcd-pki/etcd.pem + - name: ETCDCTL_ENDPOINTS + value: https://127.0.0.1:12379 + - name: ETCDCTL_KEY + value: /etc/etcd-pki/etcd-key.pem + volumeMounts: + - name: pki + mountPath: /etc/etcd-pki + readOnly: true + - name: manifests + mountPath: /etc/kubernetes/kubelet/manifests + - name: varlib + mountPath: /var/lib + volumes: + - name: data-0 + hostPath: + path: /var/lib/auxiliary-etcd-0 + - name: data-1 + hostPath: + path: /var/lib/auxiliary-etcd-1 + - name: pki + hostPath: + path: /etc/kubernetes/etcd/pki + - name: manifests + hostPath: + path: /etc/kubernetes/kubelet/manifests + - name: varlib + hostPath: + path: /var/lib