141 lines
4.7 KiB
YAML
141 lines
4.7 KiB
YAML
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: auxiliary-etcd
|
|
namespace: kube-system
|
|
labels:
|
|
application: kubernetes
|
|
component: auxiliary-etcd
|
|
promenade: genesis
|
|
spec:
|
|
hostNetwork: true
|
|
containers:
|
|
{%- with etcd_name = 'auxiliary-0', client_port = 12379, peer_port = 12380 %}
|
|
{% include "genesis-etcd/server-container.yaml" with context %}
|
|
{%- endwith %}
|
|
|
|
{%- with etcd_name = 'auxiliary-1', client_port = 22379, peer_port = 22380 %}
|
|
{% include "genesis-etcd/server-container.yaml" with context %}
|
|
{%- endwith %}
|
|
|
|
- name: monitor
|
|
image: {{ config['Genesis:images.kubernetes.etcdctl'] }}
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |-
|
|
set -x
|
|
|
|
external_member_count () {
|
|
etcdctl member list \
|
|
| grep '\bstarted\b' \
|
|
| grep -Ev "\\b(auxiliary-0|auxiliary-1)\\b" \
|
|
| wc -l
|
|
}
|
|
|
|
remove_if_possible () {
|
|
MEMBER_NAME=$1
|
|
MEMBER_ID=$(etcdctl member list | grep "${MEMBER_NAME}" | awk -F ', ' '{ print $1 }')
|
|
if [ -n "${MEMBER_ID}" ]; then
|
|
if is_leader "$MEMBER_ID"; then
|
|
abdicate
|
|
fi
|
|
etcdctl member remove $MEMBER_ID
|
|
sleep 10
|
|
fi
|
|
}
|
|
|
|
abdicate () {
|
|
NEW_LEADER=$(etcdctl member list | grep '\bstarted\b' | grep -Ev '\\b(auxiliary-0|auxiliary-1)\\b' | head -1 | awk -F ', ' '{print $1}')
|
|
if [ -n "$NEW_LEADER" ]; then
|
|
if ! etcdctl move-leader "$NEW_LEADER"; then
|
|
echo "Attempted abdication, but failed."
|
|
return
|
|
fi
|
|
sleep 5
|
|
return
|
|
fi
|
|
}
|
|
|
|
abdicate_if_needed () {
|
|
AUX_MEMBERS=$(etcdctl member list | grep '\bstarted\b' | grep -E "\\b(auxiliary-0|auxiliary-1)\\b" | awk -F ', ' '{print $1}')
|
|
for m in $AUX_MEMBERS; do
|
|
if is_leader "$m"; then
|
|
abdicate
|
|
fi
|
|
done
|
|
}
|
|
|
|
is_leader () {
|
|
MEMBER_ID="$1"
|
|
MEMBER_EP=$(etcdctl member list | grep "$MEMBER_ID" | awk -F ', ' '{print $5}')
|
|
IS_LEADER=$(ETCDCTL_ENDPOINTS="$MEMBER_EP" etcdctl endpoint status | awk -F ', ' '{ print $5 }')
|
|
if [ "$IS_LEADER" = "true" ]; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
aux_endpoint_present () {
|
|
awk '/- name: ETCD_ENDPOINTS/{getline; sub(/.*: "/, ""); sub(/".*/, ""); print}' /manifests/kubernetes-apiserver.yaml \
|
|
| grep -E "\\b(12379|22379)\\b"
|
|
}
|
|
|
|
auxiliary_threshold="{{ config.get_first('Genesis:etcd.auxiliary_threshold', default=3) }}"
|
|
# NOTE(sh8121att): If there are enough (a fully resilient contigent) non-auxiliary members,
|
|
# then we are ready to remove the auxiliary members. Otherwise, wait.
|
|
while [ ! "$(external_member_count)" -ge "$auxiliary_threshold" ] || [ "$(aux_endpoint_present)" ]; do
|
|
abdicate_if_needed
|
|
sleep 30
|
|
done
|
|
|
|
# NOTE(mark-burnett): Failures beyond this point are unexpected, but
|
|
# should be recovered by restarting this container.
|
|
set -e
|
|
|
|
remove_if_possible auxiliary-0
|
|
remove_if_possible auxiliary-1
|
|
|
|
rm -rf \
|
|
/var/lib/etcd/auxiliary-0 \
|
|
/var/lib/etcd/auxiliary-1 \
|
|
/manifests/auxiliary-kubernetes-etcd.yaml
|
|
|
|
sleep 10000
|
|
env:
|
|
- name: ETCDCTL_API
|
|
value: '3'
|
|
- name: ETCDCTL_DIAL_TIMEOUT
|
|
value: 3s
|
|
- name: ETCDCTL_ENDPOINTS
|
|
value: https://127.0.0.1:2379
|
|
- name: ETCDCTL_CACERT
|
|
value: /etc/etcd/pki/client-ca.pem
|
|
- name: ETCDCTL_CERT
|
|
value: /etc/etcd/pki/etcd-client.pem
|
|
- name: ETCDCTL_KEY
|
|
value: /etc/etcd/pki/etcd-client-key.pem
|
|
volumeMounts:
|
|
- name: all-etcd-data
|
|
mountPath: /var/lib/etcd
|
|
- name: pki-auxiliary-0
|
|
mountPath: /etc/etcd/pki
|
|
- name: manifest
|
|
mountPath: /manifests
|
|
volumes:
|
|
{%- with etcd_name = 'auxiliary-0', client_port = 12379, peer_port = 12380, volume_name = 'auxiliary-0' %}
|
|
{% include "genesis-etcd/common-volumes.yaml" with context %}
|
|
{%- endwith %}
|
|
{%- with etcd_name = 'auxiliary-1', client_port = 22379, peer_port = 22380, volume_name = 'auxiliary-1' %}
|
|
{% include "genesis-etcd/common-volumes.yaml" with context %}
|
|
{%- endwith %}
|
|
- name: manifest
|
|
hostPath:
|
|
path: /etc/kubernetes/manifests
|
|
- name: all-etcd-data
|
|
hostPath:
|
|
path: /var/lib/etcd
|
|
...
|