Merge pull request #9 from mark-burnett/cleanup-etcd-join

Use auxiliary etcd instances to smooth join process
This commit is contained in:
Mark Burnett 2017-06-20 10:45:10 -05:00 committed by GitHub
commit fac36cec4f
7 changed files with 221 additions and 74 deletions

View File

@ -46,20 +46,27 @@ def _extract_etcd_data(hostname, genesis, masters):
'env': {},
}
peers = []
for host in boot_order:
peers.append(host)
if host['hostname'] == hostname:
break
result['env']['ETCD_INITIAL_CLUSTER'] = ','.join(
'%s=https://%s:2380' % (p['hostname'], p['hostname'])
for p in peers)
peers = [
{
'hostname': 'auxiliary-etcd-%d' % i,
'peer_port': 2380 + (i + 1) * 10000
}
for i in range(2)
]
peers.append({
'hostname': genesis['hostname'],
})
if hostname == genesis['hostname']:
result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'new'
else:
result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'existing'
for host in non_genesis_masters:
peers.append({'hostname': host['hostname']})
result['env']['ETCD_INITIAL_CLUSTER'] = ','.join(
'%s=https://%s:%d' % (p['hostname'], p['hostname'], p.get('peer_port', 2380))
for p in peers)
return result

View File

@ -1,24 +0,0 @@
from . import kube, logging
__all__ = ['add_member']
LOG = logging.getLogger(__name__)
def add_member(exec_pod, hostname, port):
opts = ' '.join([
'--cacert',
'/etc/etcd-pki/cluster-ca.pem',
'--cert',
'/etc/etcd-pki/etcd.pem',
'--key',
'/etc/etcd-pki/etcd-key.pem',
])
result = kube.kc('exec', '-n', 'kube-system', '-t', exec_pod, '--', 'sh', '-c',
'ETCDCTL_API=3 etcdctl %s member add %s --peer-urls https://%s:%d'
% (opts, hostname, hostname, port))
if result.returncode != 0:
LOG.error('Failed to add etcd member. STDOUT: %r', result.stdout)
LOG.error('Failed to add etcd member. STDERR: %r', result.stderr)
result.check_returncode()

View File

@ -1,27 +0,0 @@
from . import logging
import subprocess
import time
__all__ = ['kc', 'wait_for_node']
LOG = logging.getLogger(__name__)
def wait_for_node(node):
repeat = True
while repeat:
result = kc('get', 'nodes', node, '-o',
r'jsonpath={.status.conditions[?(@.type=="Ready")].status}')
if result.stdout == b'True':
repeat = False
else:
LOG.debug('Node "%s" not ready, waiting. stdout=%r stderr=%r',
node, result.stdout, result.stderr)
time.sleep(5)
def kc(*args):
return subprocess.run(['/target/usr/local/bin/kubectl',
'--kubeconfig', '/target/etc/kubernetes/genesis/kubeconfig.yaml', *args],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)

View File

@ -1,4 +1,4 @@
from . import config, etcd, logging, kube, pki, renderer
from . import config, logging, pki, renderer
import os
import subprocess
@ -26,7 +26,6 @@ class Operator:
def genesis(self, *, asset_dir=None):
self.setup(asset_dir=asset_dir)
self.expand_etcd_cluster()
def join(self, *, asset_dir=None):
self.setup(asset_dir=asset_dir)
@ -63,15 +62,3 @@ class Operator:
self.target_dir,
'/bin/bash', '/usr/local/bin/bootstrap'],
check=True)
def expand_etcd_cluster(self):
for node in self.node_data['etcd']['boot_order'][1:]:
LOG.info('Waiting for Node "%s" to be Ready', node['hostname'])
kube.wait_for_node(node['hostname'])
LOG.info('Node "%s" Ready. Adding to etcd cluster.', node['hostname'])
etcd.add_member(self.genesis_etcd_pod, node['hostname'], port=2380)
LOG.info('Finished expanding etcd cluster.')
@property
def genesis_etcd_pod(self):
return 'kube-etcd-%s' % self.node_data['genesis']['hostname']

View File

@ -0,0 +1,2 @@
host-record=auxiliary-etcd-0,{{ genesis['ip'] }}
host-record=auxiliary-etcd-1,{{ genesis['ip'] }}

View File

@ -0,0 +1,19 @@
{
"CN": "etcd:{{ current_node['hostname'] }}",
"hosts": [
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster.local",
"127.0.0.1",
"{{ current_node['hostname'] }}",
"auxiliary-etcd-0",
"auxiliary-etcd-1",
"{{ current_node['ip'] }}",
"{{ network.kube_service_ip }}"
],
"key": {
"algo": "rsa",
"size": 2048
}
}

View File

@ -0,0 +1,183 @@
---
apiVersion: v1
kind: Pod
metadata:
name: auxiliary-etcd
namespace: kube-system
labels:
component: auxiliary-etcd
promenade: genesis
spec:
hostNetwork: true
containers:
- name: auxiliary-etcd-0
image: quay.io/coreos/etcd:v3.0.17
env:
- name: ETCD_NAME
value: auxiliary-etcd-0
- name: ETCD_CLIENT_CERT_AUTH
value: "true"
- name: ETCD_PEER_CLIENT_CERT_AUTH
value: "true"
- name: ETCD_DATA_DIR
value: /var/lib/auxiliary-etcd-0
- name: ETCD_TRUSTED_CA_FILE
value: /etc/etcd-pki/cluster-ca.pem
- name: ETCD_CERT_FILE
value: /etc/etcd-pki/etcd.pem
- name: ETCD_KEY_FILE
value: /etc/etcd-pki/etcd-key.pem
- name: ETCD_PEER_TRUSTED_CA_FILE
value: /etc/etcd-pki/cluster-ca.pem
- name: ETCD_PEER_CERT_FILE
value: /etc/etcd-pki/etcd.pem
- name: ETCD_PEER_KEY_FILE
value: /etc/etcd-pki/etcd-key.pem
- name: ETCD_ADVERTISE_CLIENT_URLS
value: https://$(ETCD_NAME):12379
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
value: https://$(ETCD_NAME):12380
- name: ETCD_INITIAL_CLUSTER_TOKEN
value: promenade-kube-etcd-token
- name: ETCD_LISTEN_CLIENT_URLS
value: https://0.0.0.0:12379
- name: ETCD_LISTEN_PEER_URLS
value: https://0.0.0.0:12380
{%- for env_name, env_value in etcd['env'].items() %}
- name: {{ env_name }}
value: {{ env_value }}
{%- endfor %}
ports:
- name: client
containerPort: 12379
- name: peer
containerPort: 12380
resources:
limits:
cpu: 100m
requests:
cpu: 100m
volumeMounts:
- name: data-0
mountPath: /var/lib/auxiliary-etcd-0
- name: pki
mountPath: /etc/etcd-pki
readOnly: true
- name: auxiliary-etcd-1
image: quay.io/coreos/etcd:v3.0.17
env:
- name: ETCD_NAME
value: auxiliary-etcd-1
- name: ETCD_CLIENT_CERT_AUTH
value: "true"
- name: ETCD_PEER_CLIENT_CERT_AUTH
value: "true"
- name: ETCD_DATA_DIR
value: /var/lib/auxiliary-etcd-1
- name: ETCD_TRUSTED_CA_FILE
value: /etc/etcd-pki/cluster-ca.pem
- name: ETCD_CERT_FILE
value: /etc/etcd-pki/etcd.pem
- name: ETCD_KEY_FILE
value: /etc/etcd-pki/etcd-key.pem
- name: ETCD_PEER_TRUSTED_CA_FILE
value: /etc/etcd-pki/cluster-ca.pem
- name: ETCD_PEER_CERT_FILE
value: /etc/etcd-pki/etcd.pem
- name: ETCD_PEER_KEY_FILE
value: /etc/etcd-pki/etcd-key.pem
- name: ETCD_ADVERTISE_CLIENT_URLS
value: https://$(ETCD_NAME):22379
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
value: https://$(ETCD_NAME):22380
- name: ETCD_INITIAL_CLUSTER_TOKEN
value: promenade-kube-etcd-token
- name: ETCD_LISTEN_CLIENT_URLS
value: https://0.0.0.0:22379
- name: ETCD_LISTEN_PEER_URLS
value: https://0.0.0.0:22380
{%- for env_name, env_value in etcd['env'].items() %}
- name: {{ env_name }}
value: {{ env_value }}
{%- endfor %}
ports:
- name: client
containerPort: 22379
- name: peer
containerPort: 22380
resources:
limits:
cpu: 100m
requests:
cpu: 100m
volumeMounts:
- name: data-1
mountPath: /var/lib/auxiliary-etcd-1
- name: pki
mountPath: /etc/etcd-pki
readOnly: true
- name: cluster-monitor
image: quay.io/coreos/etcd:v3.0.17
command:
- sh
- -c
- |-
set -x
while true; do
if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -ge {{ masters | length }} ]; then
{%- for master in masters %}
etcdctl member add {{ master['hostname'] }} --peer-urls https://{{ master['hostname'] }}:2380
{%- endfor %}
break
fi
done
while true; do
sleep 5
if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -eq {{ 2 + (masters | length) }} ]; then
etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-1 | cut -d , -f 1)
etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-0 | cut -d , -f 1)
sleep 60
rm -rf /var/lib/auxiliary-etcd-0 /var/lib/auxiliary-etcd-1 /etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml
sleep 10000
fi
done
resources:
limits:
cpu: 100m
requests:
cpu: 100m
env:
- name: ETCDCTL_API
value: "3"
- name: ETCDCTL_CACERT
value: /etc/etcd-pki/cluster-ca.pem
- name: ETCDCTL_CERT
value: /etc/etcd-pki/etcd.pem
- name: ETCDCTL_ENDPOINTS
value: https://127.0.0.1:12379
- name: ETCDCTL_KEY
value: /etc/etcd-pki/etcd-key.pem
volumeMounts:
- name: pki
mountPath: /etc/etcd-pki
readOnly: true
- name: manifests
mountPath: /etc/kubernetes/kubelet/manifests
- name: varlib
mountPath: /var/lib
volumes:
- name: data-0
hostPath:
path: /var/lib/auxiliary-etcd-0
- name: data-1
hostPath:
path: /var/lib/auxiliary-etcd-1
- name: pki
hostPath:
path: /etc/kubernetes/etcd/pki
- name: manifests
hostPath:
path: /etc/kubernetes/kubelet/manifests
- name: varlib
hostPath:
path: /var/lib