use auxiliary etcd instances to smooth join process
This commit is contained in:
parent
fce98459a6
commit
866412ce62
|
@ -46,20 +46,27 @@ def _extract_etcd_data(hostname, genesis, masters):
|
||||||
'env': {},
|
'env': {},
|
||||||
}
|
}
|
||||||
|
|
||||||
peers = []
|
peers = [
|
||||||
for host in boot_order:
|
{
|
||||||
peers.append(host)
|
'hostname': 'auxiliary-etcd-%d' % i,
|
||||||
if host['hostname'] == hostname:
|
'peer_port': 2380 + (i + 1) * 10000
|
||||||
break
|
}
|
||||||
|
for i in range(2)
|
||||||
result['env']['ETCD_INITIAL_CLUSTER'] = ','.join(
|
]
|
||||||
'%s=https://%s:2380' % (p['hostname'], p['hostname'])
|
peers.append({
|
||||||
for p in peers)
|
'hostname': genesis['hostname'],
|
||||||
|
})
|
||||||
|
|
||||||
if hostname == genesis['hostname']:
|
if hostname == genesis['hostname']:
|
||||||
result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'new'
|
result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'new'
|
||||||
else:
|
else:
|
||||||
result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'existing'
|
result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'existing'
|
||||||
|
for host in non_genesis_masters:
|
||||||
|
peers.append({'hostname': host['hostname']})
|
||||||
|
|
||||||
|
result['env']['ETCD_INITIAL_CLUSTER'] = ','.join(
|
||||||
|
'%s=https://%s:%d' % (p['hostname'], p['hostname'], p.get('peer_port', 2380))
|
||||||
|
for p in peers)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
|
@ -1,24 +0,0 @@
|
||||||
from . import kube, logging
|
|
||||||
|
|
||||||
__all__ = ['add_member']
|
|
||||||
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def add_member(exec_pod, hostname, port):
|
|
||||||
opts = ' '.join([
|
|
||||||
'--cacert',
|
|
||||||
'/etc/etcd-pki/cluster-ca.pem',
|
|
||||||
'--cert',
|
|
||||||
'/etc/etcd-pki/etcd.pem',
|
|
||||||
'--key',
|
|
||||||
'/etc/etcd-pki/etcd-key.pem',
|
|
||||||
])
|
|
||||||
result = kube.kc('exec', '-n', 'kube-system', '-t', exec_pod, '--', 'sh', '-c',
|
|
||||||
'ETCDCTL_API=3 etcdctl %s member add %s --peer-urls https://%s:%d'
|
|
||||||
% (opts, hostname, hostname, port))
|
|
||||||
if result.returncode != 0:
|
|
||||||
LOG.error('Failed to add etcd member. STDOUT: %r', result.stdout)
|
|
||||||
LOG.error('Failed to add etcd member. STDERR: %r', result.stderr)
|
|
||||||
result.check_returncode()
|
|
|
@ -1,27 +0,0 @@
|
||||||
from . import logging
|
|
||||||
import subprocess
|
|
||||||
import time
|
|
||||||
|
|
||||||
__all__ = ['kc', 'wait_for_node']
|
|
||||||
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def wait_for_node(node):
|
|
||||||
repeat = True
|
|
||||||
while repeat:
|
|
||||||
result = kc('get', 'nodes', node, '-o',
|
|
||||||
r'jsonpath={.status.conditions[?(@.type=="Ready")].status}')
|
|
||||||
if result.stdout == b'True':
|
|
||||||
repeat = False
|
|
||||||
else:
|
|
||||||
LOG.debug('Node "%s" not ready, waiting. stdout=%r stderr=%r',
|
|
||||||
node, result.stdout, result.stderr)
|
|
||||||
time.sleep(5)
|
|
||||||
|
|
||||||
|
|
||||||
def kc(*args):
|
|
||||||
return subprocess.run(['/target/usr/local/bin/kubectl',
|
|
||||||
'--kubeconfig', '/target/etc/kubernetes/genesis/kubeconfig.yaml', *args],
|
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
|
@ -1,4 +1,4 @@
|
||||||
from . import config, etcd, logging, kube, pki, renderer
|
from . import config, logging, pki, renderer
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
@ -26,7 +26,6 @@ class Operator:
|
||||||
|
|
||||||
def genesis(self, *, asset_dir=None):
|
def genesis(self, *, asset_dir=None):
|
||||||
self.setup(asset_dir=asset_dir)
|
self.setup(asset_dir=asset_dir)
|
||||||
self.expand_etcd_cluster()
|
|
||||||
|
|
||||||
def join(self, *, asset_dir=None):
|
def join(self, *, asset_dir=None):
|
||||||
self.setup(asset_dir=asset_dir)
|
self.setup(asset_dir=asset_dir)
|
||||||
|
@ -63,15 +62,3 @@ class Operator:
|
||||||
self.target_dir,
|
self.target_dir,
|
||||||
'/bin/bash', '/usr/local/bin/bootstrap'],
|
'/bin/bash', '/usr/local/bin/bootstrap'],
|
||||||
check=True)
|
check=True)
|
||||||
|
|
||||||
def expand_etcd_cluster(self):
|
|
||||||
for node in self.node_data['etcd']['boot_order'][1:]:
|
|
||||||
LOG.info('Waiting for Node "%s" to be Ready', node['hostname'])
|
|
||||||
kube.wait_for_node(node['hostname'])
|
|
||||||
LOG.info('Node "%s" Ready. Adding to etcd cluster.', node['hostname'])
|
|
||||||
etcd.add_member(self.genesis_etcd_pod, node['hostname'], port=2380)
|
|
||||||
LOG.info('Finished expanding etcd cluster.')
|
|
||||||
|
|
||||||
@property
|
|
||||||
def genesis_etcd_pod(self):
|
|
||||||
return 'kube-etcd-%s' % self.node_data['genesis']['hostname']
|
|
||||||
|
|
|
@ -0,0 +1,2 @@
|
||||||
|
host-record=auxiliary-etcd-0,{{ genesis['ip'] }}
|
||||||
|
host-record=auxiliary-etcd-1,{{ genesis['ip'] }}
|
|
@ -0,0 +1,19 @@
|
||||||
|
{
|
||||||
|
"CN": "etcd:{{ current_node['hostname'] }}",
|
||||||
|
"hosts": [
|
||||||
|
"kubernetes",
|
||||||
|
"kubernetes.default",
|
||||||
|
"kubernetes.default.svc",
|
||||||
|
"kubernetes.default.svc.cluster.local",
|
||||||
|
"127.0.0.1",
|
||||||
|
"{{ current_node['hostname'] }}",
|
||||||
|
"auxiliary-etcd-0",
|
||||||
|
"auxiliary-etcd-1",
|
||||||
|
"{{ current_node['ip'] }}",
|
||||||
|
"{{ network.kube_service_ip }}"
|
||||||
|
],
|
||||||
|
"key": {
|
||||||
|
"algo": "rsa",
|
||||||
|
"size": 2048
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,183 @@
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: auxiliary-etcd
|
||||||
|
namespace: kube-system
|
||||||
|
labels:
|
||||||
|
component: auxiliary-etcd
|
||||||
|
promenade: genesis
|
||||||
|
spec:
|
||||||
|
hostNetwork: true
|
||||||
|
containers:
|
||||||
|
- name: auxiliary-etcd-0
|
||||||
|
image: quay.io/coreos/etcd:v3.0.17
|
||||||
|
env:
|
||||||
|
- name: ETCD_NAME
|
||||||
|
value: auxiliary-etcd-0
|
||||||
|
- name: ETCD_CLIENT_CERT_AUTH
|
||||||
|
value: "true"
|
||||||
|
- name: ETCD_PEER_CLIENT_CERT_AUTH
|
||||||
|
value: "true"
|
||||||
|
- name: ETCD_DATA_DIR
|
||||||
|
value: /var/lib/auxiliary-etcd-0
|
||||||
|
- name: ETCD_TRUSTED_CA_FILE
|
||||||
|
value: /etc/etcd-pki/cluster-ca.pem
|
||||||
|
- name: ETCD_CERT_FILE
|
||||||
|
value: /etc/etcd-pki/etcd.pem
|
||||||
|
- name: ETCD_KEY_FILE
|
||||||
|
value: /etc/etcd-pki/etcd-key.pem
|
||||||
|
- name: ETCD_PEER_TRUSTED_CA_FILE
|
||||||
|
value: /etc/etcd-pki/cluster-ca.pem
|
||||||
|
- name: ETCD_PEER_CERT_FILE
|
||||||
|
value: /etc/etcd-pki/etcd.pem
|
||||||
|
- name: ETCD_PEER_KEY_FILE
|
||||||
|
value: /etc/etcd-pki/etcd-key.pem
|
||||||
|
- name: ETCD_ADVERTISE_CLIENT_URLS
|
||||||
|
value: https://$(ETCD_NAME):12379
|
||||||
|
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
|
||||||
|
value: https://$(ETCD_NAME):12380
|
||||||
|
- name: ETCD_INITIAL_CLUSTER_TOKEN
|
||||||
|
value: promenade-kube-etcd-token
|
||||||
|
- name: ETCD_LISTEN_CLIENT_URLS
|
||||||
|
value: https://0.0.0.0:12379
|
||||||
|
- name: ETCD_LISTEN_PEER_URLS
|
||||||
|
value: https://0.0.0.0:12380
|
||||||
|
{%- for env_name, env_value in etcd['env'].items() %}
|
||||||
|
- name: {{ env_name }}
|
||||||
|
value: {{ env_value }}
|
||||||
|
{%- endfor %}
|
||||||
|
ports:
|
||||||
|
- name: client
|
||||||
|
containerPort: 12379
|
||||||
|
- name: peer
|
||||||
|
containerPort: 12380
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
volumeMounts:
|
||||||
|
- name: data-0
|
||||||
|
mountPath: /var/lib/auxiliary-etcd-0
|
||||||
|
- name: pki
|
||||||
|
mountPath: /etc/etcd-pki
|
||||||
|
readOnly: true
|
||||||
|
- name: auxiliary-etcd-1
|
||||||
|
image: quay.io/coreos/etcd:v3.0.17
|
||||||
|
env:
|
||||||
|
- name: ETCD_NAME
|
||||||
|
value: auxiliary-etcd-1
|
||||||
|
- name: ETCD_CLIENT_CERT_AUTH
|
||||||
|
value: "true"
|
||||||
|
- name: ETCD_PEER_CLIENT_CERT_AUTH
|
||||||
|
value: "true"
|
||||||
|
- name: ETCD_DATA_DIR
|
||||||
|
value: /var/lib/auxiliary-etcd-1
|
||||||
|
- name: ETCD_TRUSTED_CA_FILE
|
||||||
|
value: /etc/etcd-pki/cluster-ca.pem
|
||||||
|
- name: ETCD_CERT_FILE
|
||||||
|
value: /etc/etcd-pki/etcd.pem
|
||||||
|
- name: ETCD_KEY_FILE
|
||||||
|
value: /etc/etcd-pki/etcd-key.pem
|
||||||
|
- name: ETCD_PEER_TRUSTED_CA_FILE
|
||||||
|
value: /etc/etcd-pki/cluster-ca.pem
|
||||||
|
- name: ETCD_PEER_CERT_FILE
|
||||||
|
value: /etc/etcd-pki/etcd.pem
|
||||||
|
- name: ETCD_PEER_KEY_FILE
|
||||||
|
value: /etc/etcd-pki/etcd-key.pem
|
||||||
|
- name: ETCD_ADVERTISE_CLIENT_URLS
|
||||||
|
value: https://$(ETCD_NAME):22379
|
||||||
|
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
|
||||||
|
value: https://$(ETCD_NAME):22380
|
||||||
|
- name: ETCD_INITIAL_CLUSTER_TOKEN
|
||||||
|
value: promenade-kube-etcd-token
|
||||||
|
- name: ETCD_LISTEN_CLIENT_URLS
|
||||||
|
value: https://0.0.0.0:22379
|
||||||
|
- name: ETCD_LISTEN_PEER_URLS
|
||||||
|
value: https://0.0.0.0:22380
|
||||||
|
{%- for env_name, env_value in etcd['env'].items() %}
|
||||||
|
- name: {{ env_name }}
|
||||||
|
value: {{ env_value }}
|
||||||
|
{%- endfor %}
|
||||||
|
ports:
|
||||||
|
- name: client
|
||||||
|
containerPort: 22379
|
||||||
|
- name: peer
|
||||||
|
containerPort: 22380
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
volumeMounts:
|
||||||
|
- name: data-1
|
||||||
|
mountPath: /var/lib/auxiliary-etcd-1
|
||||||
|
- name: pki
|
||||||
|
mountPath: /etc/etcd-pki
|
||||||
|
readOnly: true
|
||||||
|
- name: cluster-monitor
|
||||||
|
image: quay.io/coreos/etcd:v3.0.17
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- -c
|
||||||
|
- |-
|
||||||
|
set -x
|
||||||
|
while true; do
|
||||||
|
if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -ge {{ masters | length }} ]; then
|
||||||
|
{%- for master in masters %}
|
||||||
|
etcdctl member add {{ master['hostname'] }} --peer-urls https://{{ master['hostname'] }}:2380
|
||||||
|
{%- endfor %}
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
while true; do
|
||||||
|
sleep 5
|
||||||
|
if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -eq {{ 2 + (masters | length) }} ]; then
|
||||||
|
etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-1 | cut -d , -f 1)
|
||||||
|
etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-0 | cut -d , -f 1)
|
||||||
|
sleep 60
|
||||||
|
rm -rf /var/lib/auxiliary-etcd-0 /var/lib/auxiliary-etcd-1 /etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml
|
||||||
|
sleep 10000
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
env:
|
||||||
|
- name: ETCDCTL_API
|
||||||
|
value: "3"
|
||||||
|
- name: ETCDCTL_CACERT
|
||||||
|
value: /etc/etcd-pki/cluster-ca.pem
|
||||||
|
- name: ETCDCTL_CERT
|
||||||
|
value: /etc/etcd-pki/etcd.pem
|
||||||
|
- name: ETCDCTL_ENDPOINTS
|
||||||
|
value: https://127.0.0.1:12379
|
||||||
|
- name: ETCDCTL_KEY
|
||||||
|
value: /etc/etcd-pki/etcd-key.pem
|
||||||
|
volumeMounts:
|
||||||
|
- name: pki
|
||||||
|
mountPath: /etc/etcd-pki
|
||||||
|
readOnly: true
|
||||||
|
- name: manifests
|
||||||
|
mountPath: /etc/kubernetes/kubelet/manifests
|
||||||
|
- name: varlib
|
||||||
|
mountPath: /var/lib
|
||||||
|
volumes:
|
||||||
|
- name: data-0
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/auxiliary-etcd-0
|
||||||
|
- name: data-1
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/auxiliary-etcd-1
|
||||||
|
- name: pki
|
||||||
|
hostPath:
|
||||||
|
path: /etc/kubernetes/etcd/pki
|
||||||
|
- name: manifests
|
||||||
|
hostPath:
|
||||||
|
path: /etc/kubernetes/kubelet/manifests
|
||||||
|
- name: varlib
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib
|
Loading…
Reference in New Issue