---
apiVersion: v1
kind: Pod
metadata:
  name: auxiliary-etcd
  namespace: kube-system
  labels:
    application: kubernetes
    component: auxiliary-etcd
    promenade: genesis
spec:
  hostNetwork: true
  containers:
{%- with etcd_name = 'auxiliary-0', client_port = 12379, peer_port = 12380  %}
{% include "genesis-etcd/server-container.yaml" with context %}
{%- endwith %}

{%- with etcd_name = 'auxiliary-1', client_port = 22379, peer_port = 22380  %}
{% include "genesis-etcd/server-container.yaml" with context %}
{%- endwith %}

    - name: monitor
      image: {{ config['Genesis:images.kubernetes.etcdctl'] }}
      command:
        - /bin/sh
        - -c
        - |-
          set -x

          external_member_count () {
              etcdctl member list \
                  | grep '\bstarted\b' \
                  | grep -Ev "\\b(auxiliary-0|auxiliary-1)\\b" \
                  | wc -l
          }

          remove_if_possible () {
              MEMBER_NAME="$1"
              MEMBER_ID=$(etcdctl member list | grep "${MEMBER_NAME}" | awk -F ', ' '{ print $1 }')
              if [ -n "${MEMBER_ID}" ]; then
                  if is_leader "$MEMBER_ID"; then
                    abdicate "$MEMBER_ID"
                  fi
                  etcdctl member remove $MEMBER_ID
                  sleep 5
              fi
          }

          abdicate () {
            OLD_LEADER="$1"
            OLD_LEADER_EP=$(etcdctl member list | grep "$OLD_LEADER" | awk -F ', ' '{print $5}')
            NEW_LEADER=$(etcdctl member list | grep '\bstarted\b' | grep -Ev "\\b(auxiliary-0|auxiliary-1)\\b" | head -1 | awk -F ', ' '{print $1}')
            if [ -n "$NEW_LEADER" ]; then
              if ! ETCDCTL_ENDPOINTS="$OLD_LEADER_EP" etcdctl move-leader "$NEW_LEADER"; then
                echo "Attempted abdication, but failed."
                return
              fi
              sleep 5
              return
            fi
          }

          abdicate_if_needed () {
            AUX_MEMBERS=$(etcdctl member list | grep '\bstarted\b' | grep -E "\\b(auxiliary-0|auxiliary-1)\\b" | awk -F ', ' '{print $1}')
            for m in $AUX_MEMBERS; do
              if is_leader "$m"; then
                abdicate "$m"
              fi
            done
          }

          is_leader () {
            MEMBER_ID="$1"
            MEMBER_EP=$(etcdctl member list | grep "$MEMBER_ID" | awk -F ', ' '{print $5}')
            IS_LEADER=$(ETCDCTL_ENDPOINTS="$MEMBER_EP" etcdctl endpoint status | awk -F ', ' '{ print $5 }')
            if [ "$IS_LEADER" = "true" ]; then
              return 0
            else
              return 1
            fi
          }

          aux_endpoint_present () {
            awk '/- name: ETCD_ENDPOINTS/{getline; sub(/.*: "/, ""); sub(/".*/, ""); print}' /manifests/kubernetes-apiserver.yaml \
                          | grep -E "\\b(12379|22379)\\b"
          }

          auxiliary_threshold="{{ config.get_first('Genesis:etcd.auxiliary_threshold', default=3) }}"
          # NOTE(sh8121att): If there are enough (a fully resilient contigent) non-auxiliary members,
          # then we are ready to remove the auxiliary members.  Otherwise, wait.
          while [ ! "$(external_member_count)" -ge "$auxiliary_threshold" ] || [ "$(aux_endpoint_present)" ]; do
              abdicate_if_needed
              sleep 30
          done

          # NOTE(mark-burnett): Failures beyond this point are unexpected, but
          # should be recovered by restarting this container.
          set -e

          remove_if_possible auxiliary-0
          remove_if_possible auxiliary-1

          rm -rf \
              /var/lib/etcd/auxiliary-0 \
              /var/lib/etcd/auxiliary-1 \
              /manifests/auxiliary-kubernetes-etcd.yaml

          sleep 10000
      env:
        - name: ETCDCTL_API
          value: '3'
        - name: ETCDCTL_DIAL_TIMEOUT
          value: 3s
        - name: ETCDCTL_ENDPOINTS
          value: https://127.0.0.1:2379
        - name: ETCDCTL_CACERT
          value: /etc/etcd/pki/client-ca.pem
        - name: ETCDCTL_CERT
          value: /etc/etcd/pki/etcd-client.pem
        - name: ETCDCTL_KEY
          value: /etc/etcd/pki/etcd-client-key.pem
      volumeMounts:
        - name: all-etcd-data
          mountPath: /var/lib/etcd
        - name: pki-auxiliary-0
          mountPath: /etc/etcd/pki
        - name: manifest
          mountPath: /manifests
  volumes:
{%- with etcd_name = 'auxiliary-0', client_port = 12379, peer_port = 12380, volume_name = 'auxiliary-0' %}
{% include "genesis-etcd/common-volumes.yaml" with context %}
{%- endwith %}
{%- with etcd_name = 'auxiliary-1', client_port = 22379, peer_port = 22380, volume_name = 'auxiliary-1' %}
{% include "genesis-etcd/common-volumes.yaml" with context %}
{%- endwith %}
    - name: manifest
      hostPath:
        path: /etc/kubernetes/manifests
    - name: all-etcd-data
      hostPath:
        path: /var/lib/etcd
...