From e6f1b7de8352c1e1a18d6d2be8f736547062c046 Mon Sep 17 00:00:00 2001
From: Ruslan Aliev <raliev@mirantis.com>
Date: Thu, 18 Apr 2024 19:05:18 -0500
Subject: [PATCH] Remove healthcheck sidecar from etcd

Also make a switch from auxiliary to permanent cluster.

Signed-off-by: Ruslan Aliev <raliev@mirantis.com>
Change-Id: I7918072a6ba5a6b22b359d1616def8c31425462d
---
 .../templates/etc/_kubernetes-etcd.yaml.tpl   | 49 ++++---------------
 charts/etcd/values.yaml                       |  7 ---
 .../manifests/auxiliary-kubernetes-etcd.yaml  |  6 +++
 3 files changed, 16 insertions(+), 46 deletions(-)

diff --git a/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl b/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl
index 5df4eb10..6dd80e08 100644
--- a/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl
+++ b/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl
@@ -16,22 +16,16 @@
 
 {{- $envAll := . }}
 {{- define "etcdreadinessProbeTemplate" }}
-exec:
-  command:
-    - /bin/sh
-    - -c
-    - |-
-      etcdctl endpoint health
-      exit $?
+httpGet:
+  path: /readyz
+  port: {{ .Values.network.service_client.port }}
+  scheme: HTTPS
 {{- end }}
 {{- define "etcdlivenessProbeTemplate" }}
-exec:
-  command:
-    - /bin/sh
-    - -c
-    - |-
-      etcdctl endpoint status
-      exit $?
+httpGet:
+  path: /livez
+  port: {{ .Values.network.service_client.port }}
+  scheme: HTTPS
 {{- end }}
 # Strip off "etcd" from service name to get the application name
 # Note that application can either be kubernetes or calico for now
@@ -123,34 +117,11 @@ spec:
         - name: MANIFEST_PATH
           value: /manifests/{{ .Values.service.name }}.yaml
 {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.etcd | indent 8 }}
-      volumeMounts:
-        - name: data
-          mountPath: /var/lib/etcd
-        - name: etc
-          mountPath: /etc/etcd
-    - name: etcd-health-check
-      image: {{ .Values.images.tags.etcdctl }}
-      imagePullPolicy: {{ .Values.images.pull_policy }}
-{{ tuple $envAll $envAll.Values.pod.resources.etcd_pod_sidecar | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
-{{ dict "envAll" $envAll "application" "etcd" "container" "etcd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }}
-      env:
-        - name: ETCDCTL_API
-          value: "{{ .Values.etcd.etcdctl_api }}"
-        - name: ETCDCTL_DIAL_TIMEOUT
-          value: "3s"
-        - name: ETCDCTL_ENDPOINTS
-          value: "https://127.0.0.1:{{ .Values.network.service_client.target_port }}"
-        - name: ETCDCTL_CACERT
-          value: "/etc/etcd/tls/client-ca.pem"
-        - name: ETCDCTL_CERT
-          value: "/etc/etcd/tls/etcd-client.pem"
-        - name: ETCDCTL_KEY
-          value: "/etc/etcd/tls/etcd-client-key.pem"
-      command: ["/bin/sh", "-c", "--"]
-      args: ["while true; do sleep 30; done;"]
 {{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "readiness" "probeTemplate" (include "etcdreadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }}
 {{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "liveness" "probeTemplate" (include "etcdlivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }}
       volumeMounts:
+        - name: data
+          mountPath: /var/lib/etcd
         - name: etc
           mountPath: /etc/etcd
   volumes:
diff --git a/charts/etcd/values.yaml b/charts/etcd/values.yaml
index 739db2c8..389b1b93 100644
--- a/charts/etcd/values.yaml
+++ b/charts/etcd/values.yaml
@@ -261,13 +261,6 @@ pod:
       limits:
         memory: "1024Mi"
         cpu: "2000m"
-    etcd_pod_sidecar:
-      requests:
-        memory: "128Mi"
-        cpu: "100m"
-      limits:
-        memory: "1024Mi"
-        cpu: "2000m"
     test:
       limits:
         memory: "128Mi"
diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml
index 6ee6e770..3897bac7 100644
--- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml
+++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml
@@ -39,6 +39,7 @@ spec:
               MEMBER_ID=$(etcdctl member list | grep "${MEMBER_NAME}" | awk -F ', ' '{ print $1 }')
               if [ -n "${MEMBER_ID}" ]; then
                   etcdctl member remove $MEMBER_ID
+                  sleep 10 
               fi
           }
 
@@ -53,6 +54,11 @@ spec:
           # should be recovered by restarting this container.
           set -e
 
+          sleep 600
+          # make a switch to permanent etcd cluster
+          ENDPOINTS=$(etcdctl member list | awk -F ',' '$3 ~! /auxiliary/ { if (output != "") output=output","$5; else output=$5} END {gsub(/ /, "", output); print output}' | sed 's/\//\\\//g')
+          sed -i "/- name: ETCD_ENDPOINTS/{n;s/value: .*/value: \"$ENDPOINTS\"/}" /manifests/kubernetes-apiserver.yaml
+
           remove_if_possible auxiliary-0
           remove_if_possible auxiliary-1