use auxiliary etcd instances to smooth join process

promenade/config.py

@@ -46,20 +46,27 @@ def _extract_etcd_data(hostname, genesis, masters):
         'env': {},
     }
 
-    peers = []
-    for host in boot_order:
-        peers.append(host)
-        if host['hostname'] == hostname:
-            break
-
-    result['env']['ETCD_INITIAL_CLUSTER'] = ','.join(
-            '%s=https://%s:2380' % (p['hostname'], p['hostname'])
-            for p in peers)
+    peers = [
+        {
+            'hostname': 'auxiliary-etcd-%d' % i,
+            'peer_port': 2380 + (i + 1) * 10000
+        }
+        for i in range(2)
+    ]
+    peers.append({
+        'hostname': genesis['hostname'],
+    })
 
     if hostname == genesis['hostname']:
         result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'new'
     else:
         result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'existing'
+        for host in non_genesis_masters:
+            peers.append({'hostname': host['hostname']})
+
+    result['env']['ETCD_INITIAL_CLUSTER'] = ','.join(
+            '%s=https://%s:%d' % (p['hostname'], p['hostname'], p.get('peer_port', 2380))
+            for p in peers)
 
     return result
 

promenade/etcd.py

@@ -1,24 +0,0 @@
-from . import kube, logging
-
-__all__ = ['add_member']
-
-
-LOG = logging.getLogger(__name__)
-
-
-def add_member(exec_pod, hostname, port):
-    opts = ' '.join([
-        '--cacert',
-        '/etc/etcd-pki/cluster-ca.pem',
-        '--cert',
-        '/etc/etcd-pki/etcd.pem',
-        '--key',
-        '/etc/etcd-pki/etcd-key.pem',
-    ])
-    result = kube.kc('exec', '-n', 'kube-system', '-t', exec_pod, '--', 'sh', '-c',
-                     'ETCDCTL_API=3 etcdctl %s member add %s --peer-urls https://%s:%d'
-                     % (opts, hostname, hostname, port))
-    if result.returncode != 0:
-        LOG.error('Failed to add etcd member. STDOUT: %r', result.stdout)
-        LOG.error('Failed to add etcd member. STDERR: %r', result.stderr)
-        result.check_returncode()

promenade/kube.py

@@ -1,27 +0,0 @@
-from . import logging
-import subprocess
-import time
-
-__all__ = ['kc', 'wait_for_node']
-
-
-LOG = logging.getLogger(__name__)
-
-
-def wait_for_node(node):
-    repeat = True
-    while repeat:
-        result = kc('get', 'nodes', node, '-o',
-                    r'jsonpath={.status.conditions[?(@.type=="Ready")].status}')
-        if result.stdout == b'True':
-            repeat = False
-        else:
-            LOG.debug('Node "%s" not ready, waiting. stdout=%r stderr=%r',
-                      node, result.stdout, result.stderr)
-            time.sleep(5)
-
-
-def kc(*args):
-    return subprocess.run(['/target/usr/local/bin/kubectl',
-        '--kubeconfig', '/target/etc/kubernetes/genesis/kubeconfig.yaml', *args],
-        stdout=subprocess.PIPE, stderr=subprocess.PIPE)

promenade/operator.py

@@ -1,4 +1,4 @@
-from . import config, etcd, logging, kube, pki, renderer
+from . import config, logging, pki, renderer
 import os
 import subprocess
 
@@ -26,7 +26,6 @@ class Operator:
 
     def genesis(self, *, asset_dir=None):
         self.setup(asset_dir=asset_dir)
-        self.expand_etcd_cluster()
 
     def join(self, *, asset_dir=None):
         self.setup(asset_dir=asset_dir)
@@ -63,15 +62,3 @@ class Operator:
                         self.target_dir,
                         '/bin/bash', '/usr/local/bin/bootstrap'],
                        check=True)
-
-    def expand_etcd_cluster(self):
-        for node in self.node_data['etcd']['boot_order'][1:]:
-            LOG.info('Waiting for Node "%s" to be Ready', node['hostname'])
-            kube.wait_for_node(node['hostname'])
-            LOG.info('Node "%s" Ready.  Adding to etcd cluster.', node['hostname'])
-            etcd.add_member(self.genesis_etcd_pod, node['hostname'], port=2380)
-        LOG.info('Finished expanding etcd cluster.')
-
-    @property
-    def genesis_etcd_pod(self):
-        return 'kube-etcd-%s' % self.node_data['genesis']['hostname']

promenade/templates/common/etc/dnsmasq.d/auxilliary-etcd

@@ -0,0 +1,2 @@
+host-record=auxiliary-etcd-0,{{ genesis['ip'] }}
+host-record=auxiliary-etcd-1,{{ genesis['ip'] }}

promenade/templates/genesis/etc/kubernetes/cfssl/csr-configs/etcd.json

@@ -0,0 +1,19 @@
+{
+  "CN": "etcd:{{ current_node['hostname'] }}",
+  "hosts": [
+    "kubernetes",
+    "kubernetes.default",
+    "kubernetes.default.svc",
+    "kubernetes.default.svc.cluster.local",
+    "127.0.0.1",
+    "{{ current_node['hostname'] }}",
+    "auxiliary-etcd-0",
+    "auxiliary-etcd-1",
+    "{{ current_node['ip'] }}",
+    "{{ network.kube_service_ip }}"
+  ],
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  }
+}

promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxilliary-etcd.yaml

@@ -0,0 +1,183 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: auxiliary-etcd
+  namespace: kube-system
+  labels:
+    component: auxiliary-etcd
+    promenade: genesis
+spec:
+  hostNetwork: true
+  containers:
+    - name: auxiliary-etcd-0
+      image: quay.io/coreos/etcd:v3.0.17
+      env:
+        - name: ETCD_NAME
+          value: auxiliary-etcd-0
+        - name: ETCD_CLIENT_CERT_AUTH
+          value: "true"
+        - name: ETCD_PEER_CLIENT_CERT_AUTH
+          value: "true"
+        - name: ETCD_DATA_DIR
+          value: /var/lib/auxiliary-etcd-0
+        - name: ETCD_TRUSTED_CA_FILE
+          value: /etc/etcd-pki/cluster-ca.pem
+        - name: ETCD_CERT_FILE
+          value: /etc/etcd-pki/etcd.pem
+        - name: ETCD_KEY_FILE
+          value: /etc/etcd-pki/etcd-key.pem
+        - name: ETCD_PEER_TRUSTED_CA_FILE
+          value: /etc/etcd-pki/cluster-ca.pem
+        - name: ETCD_PEER_CERT_FILE
+          value: /etc/etcd-pki/etcd.pem
+        - name: ETCD_PEER_KEY_FILE
+          value: /etc/etcd-pki/etcd-key.pem
+        - name: ETCD_ADVERTISE_CLIENT_URLS
+          value: https://$(ETCD_NAME):12379
+        - name: ETCD_INITIAL_ADVERTISE_PEER_URLS
+          value: https://$(ETCD_NAME):12380
+        - name: ETCD_INITIAL_CLUSTER_TOKEN
+          value: promenade-kube-etcd-token
+        - name: ETCD_LISTEN_CLIENT_URLS
+          value: https://0.0.0.0:12379
+        - name: ETCD_LISTEN_PEER_URLS
+          value: https://0.0.0.0:12380
+{%- for env_name, env_value in etcd['env'].items() %}
+        - name: {{ env_name }}
+          value: {{ env_value }}
+{%- endfor %}
+      ports:
+        - name: client
+          containerPort: 12379
+        - name: peer
+          containerPort: 12380
+      resources:
+        limits:
+          cpu: 100m
+        requests:
+          cpu: 100m
+      volumeMounts:
+        - name: data-0
+          mountPath: /var/lib/auxiliary-etcd-0
+        - name: pki
+          mountPath: /etc/etcd-pki
+          readOnly: true
+    - name: auxiliary-etcd-1
+      image: quay.io/coreos/etcd:v3.0.17
+      env:
+        - name: ETCD_NAME
+          value: auxiliary-etcd-1
+        - name: ETCD_CLIENT_CERT_AUTH
+          value: "true"
+        - name: ETCD_PEER_CLIENT_CERT_AUTH
+          value: "true"
+        - name: ETCD_DATA_DIR
+          value: /var/lib/auxiliary-etcd-1
+        - name: ETCD_TRUSTED_CA_FILE
+          value: /etc/etcd-pki/cluster-ca.pem
+        - name: ETCD_CERT_FILE
+          value: /etc/etcd-pki/etcd.pem
+        - name: ETCD_KEY_FILE
+          value: /etc/etcd-pki/etcd-key.pem
+        - name: ETCD_PEER_TRUSTED_CA_FILE
+          value: /etc/etcd-pki/cluster-ca.pem
+        - name: ETCD_PEER_CERT_FILE
+          value: /etc/etcd-pki/etcd.pem
+        - name: ETCD_PEER_KEY_FILE
+          value: /etc/etcd-pki/etcd-key.pem
+        - name: ETCD_ADVERTISE_CLIENT_URLS
+          value: https://$(ETCD_NAME):22379
+        - name: ETCD_INITIAL_ADVERTISE_PEER_URLS
+          value: https://$(ETCD_NAME):22380
+        - name: ETCD_INITIAL_CLUSTER_TOKEN
+          value: promenade-kube-etcd-token
+        - name: ETCD_LISTEN_CLIENT_URLS
+          value: https://0.0.0.0:22379
+        - name: ETCD_LISTEN_PEER_URLS
+          value: https://0.0.0.0:22380
+{%- for env_name, env_value in etcd['env'].items() %}
+        - name: {{ env_name }}
+          value: {{ env_value }}
+{%- endfor %}
+      ports:
+        - name: client
+          containerPort: 22379
+        - name: peer
+          containerPort: 22380
+      resources:
+        limits:
+          cpu: 100m
+        requests:
+          cpu: 100m
+      volumeMounts:
+        - name: data-1
+          mountPath: /var/lib/auxiliary-etcd-1
+        - name: pki
+          mountPath: /etc/etcd-pki
+          readOnly: true
+    - name: cluster-monitor
+      image: quay.io/coreos/etcd:v3.0.17
+      command:
+        - sh
+        - -c
+        - |-
+            set -x
+            while true; do
+              if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -ge {{ masters | length }} ]; then
+                {%- for master in masters %}
+                etcdctl member add {{ master['hostname'] }} --peer-urls https://{{ master['hostname'] }}:2380
+                {%- endfor %}
+                break
+              fi
+            done
+            while true; do
+              sleep 5
+              if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -eq {{ 2 + (masters | length) }} ]; then
+                etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-1 | cut -d , -f 1)
+                etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-0 | cut -d , -f 1)
+                sleep 60
+                rm -rf /var/lib/auxiliary-etcd-0 /var/lib/auxiliary-etcd-1 /etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml
+                sleep 10000
+              fi
+            done
+      resources:
+        limits:
+          cpu: 100m
+        requests:
+          cpu: 100m
+      env:
+        - name: ETCDCTL_API
+          value: "3"
+        - name: ETCDCTL_CACERT
+          value: /etc/etcd-pki/cluster-ca.pem
+        - name: ETCDCTL_CERT
+          value: /etc/etcd-pki/etcd.pem
+        - name: ETCDCTL_ENDPOINTS
+          value: https://127.0.0.1:12379
+        - name: ETCDCTL_KEY
+          value: /etc/etcd-pki/etcd-key.pem
+      volumeMounts:
+        - name: pki
+          mountPath: /etc/etcd-pki
+          readOnly: true
+        - name: manifests
+          mountPath: /etc/kubernetes/kubelet/manifests
+        - name: varlib
+          mountPath: /var/lib
+  volumes:
+    - name: data-0
+      hostPath:
+        path: /var/lib/auxiliary-etcd-0
+    - name: data-1
+      hostPath:
+        path: /var/lib/auxiliary-etcd-1
+    - name: pki
+      hostPath:
+        path: /etc/kubernetes/etcd/pki
+    - name: manifests
+      hostPath:
+        path: /etc/kubernetes/kubelet/manifests
+    - name: varlib
+      hostPath:
+        path: /var/lib