Adjusting daemonset anchor readiness check

To avoid pods cycling too quickly by checking if manifest
was created by daemonset and the component on the same host
is ready

Change-Id: I7f9b35e222ef5934fca71f30fdf9941caa60ccd7

charts/controller_manager/Chart.yaml

@@ -15,4 +15,4 @@
 apiVersion: v1
 description: A chart for Kubernetes controller-manager
 name: controller_manager
-version: 0.1.1
+version: 0.1.2

charts/controller_manager/templates/bin/_anchor.tpl

@@ -15,24 +15,60 @@
 
 set -xu
 
-compare_copy_files() {
+snapshot_files() {
+    SNAPSHOT_DIR=${1}
+    {{ range $dest, $source := .Values.anchor.files_to_copy }}
+    mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
+    cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
+    {{- end }}
+    {{ range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/controller-manager/{{ $val.file }}"
+    {{- end }}
+    {{- end }}
+    # annotate the static manifest with the name of the creating anchor pod
+    sed -i "/created-by: /s/ANCHOR_POD/${POD_NAME}/" "${SNAPSHOT_DIR}{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-controller-manager.yaml"
+}
 
-    {{range .Values.anchor.files_to_copy}}
+compare_copy_files() {
-    if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
+    SNAPSHOT_DIR=${1}
-        mkdir -p $(dirname /host{{ .dest }})
+    {{ range $dest, $source := .Values.anchor.files_to_copy }}
-        cp {{ .source }} /host{{ .dest }}
+    SRC="${SNAPSHOT_DIR}{{ $dest }}"
-        chmod go-rwx /host{{ .dest }}
+    DEST="/host{{ $dest }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
     fi
-    {{end}}
+    {{- end}}
+    {{ range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    SRC="${SNAPSHOT_DIR}/etc/kubernetes/controller-manager/{{ $val.file }}"
+    DEST="/host/etc/kubernetes/controller-manager/{{ $val.file }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
+    fi
+    {{- end }}
+    {{- end }}
 }
 
 cleanup() {
-
+    {{- range $dest, $source := .Values.anchor.files_to_copy }}
-    {{range .Values.anchor.files_to_copy}}
+    rm -f "/host{{ $dest }}"
-    rm -f /host{{ .dest }}
+    {{- end }}
-    {{end}}
+    {{  range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    rm -f "/host/etc/kubernetes/controller-manager/{{ $val.file }}"
+    {{- end }}
+    {{- end }}
 }
 
+SNAPSHOT_DIR=$(mktemp -d)
+
+snapshot_files "${SNAPSHOT_DIR}"
+
 while true; do
 
     if [ -e /tmp/stop ]; then
@@ -45,7 +81,7 @@ while true; do
 
     # Compare and replace files on Genesis host if needed
     # Copy files to other master nodes
-    compare_copy_files
+    compare_copy_files "${SNAPSHOT_DIR}"
 
     sleep {{ .Values.anchor.period }}
 done

charts/controller_manager/templates/daemonset.yaml

@@ -64,6 +64,10 @@ spec:
             value: /host{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-controller-manager.yaml
           - name: ETC_PATH
             value: /host{{ .Values.controller_manager.host_etc_path }}
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
 {{ tuple $envAll $envAll.Values.pod.resources.anchor_pod | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "kubernetes" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
@@ -73,13 +77,20 @@ spec:
               exec:
                 command:
                   - /tmp/bin/pre_stop
-
           readinessProbe:
-            httpGet:
+            exec:
-              host: 127.0.0.1
+              command:
-              path: /healthz
+              - /bin/bash
-              port: {{ .Values.network.kubernetes_controller_manager.port }}
+              - -c
-              scheme: HTTPS
+              - |-
+                grep -q "created-by: ${POD_NAME}" "${MANIFEST_PATH}" || exit 1
+                [ "$(curl -k -s -S -o /dev/null \
+                    --cert "/host{{ .Values.controller_manager.host_etc_path }}/controller-manager.pem" \
+                    --key "/host{{ .Values.controller_manager.host_etc_path }}/controller-manager-key.pem" \
+                    --cacert "/host{{ .Values.controller_manager.host_etc_path }}/cluster-ca.pem" \
+                    "https://localhost:{{ .Values.network.kubernetes_controller_manager.port }}/healthz" \
+                    -w "%{http_code}")" = "200" ]
+                exit $?
             initialDelaySeconds: 10
             periodSeconds: 5
             timeoutSeconds: 5

charts/controller_manager/templates/etc/_kubernetes-controller-manager.yaml.tpl

@@ -23,6 +23,7 @@ metadata:
     {{ .Values.service.name }}-service: enabled
 {{ tuple $envAll "kubernetes" "controller-manager" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
   annotations:
+    created-by: ANCHOR_POD
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
 {{ dict "envAll" $envAll "podName" "controller-manager" "containerNames" (list "controller-manager") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }}
 spec:

charts/controller_manager/values.yaml

@@ -32,18 +32,12 @@ anchor:
     manifest_path: /etc/kubernetes/manifests
   period: 15
   files_to_copy:
-    - source: /configmap/cluster-ca.pem
+    /etc/kubernetes/controller-manager/cluster-ca.pem: /configmap/cluster-ca.pem
-      dest: /etc/kubernetes/controller-manager/cluster-ca.pem
+    /etc/kubernetes/controller-manager/controller-manager.pem: /configmap/controller-manager.pem
-    - source: /configmap/controller-manager.pem
+    /etc/kubernetes/controller-manager/kubeconfig.yaml: /configmap/kubeconfig.yaml
-      dest: /etc/kubernetes/controller-manager/controller-manager.pem
+    /etc/kubernetes/controller-manager/controller-manager-key.pem: /secret/controller-manager-key.pem
-    - source: /configmap/kubeconfig.yaml
+    /etc/kubernetes/controller-manager/service-account.priv: /secret/service-account.priv
-      dest: /etc/kubernetes/controller-manager/kubeconfig.yaml
+    /etc/kubernetes/manifests/kubernetes-controller-manager.yaml: /configmap/kubernetes-controller-manager.yaml
-    - source: /secret/controller-manager-key.pem
-      dest: /etc/kubernetes/controller-manager/controller-manager-key.pem
-    - source: /secret/service-account.priv
-      dest: /etc/kubernetes/controller-manager/service-account.priv
-    - source: /configmap/kubernetes-controller-manager.yaml
-      dest: /etc/kubernetes/manifests/kubernetes-controller-manager.yaml
 
 controller_manager:
   host_etc_path: /etc/kubernetes/controller-manager

charts/scheduler/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 description: A chart for Kubernetes scheduler.
 name: scheduler
-version: 0.1.1
+version: 0.1.2

charts/scheduler/templates/bin/_anchor.tpl

@@ -17,22 +17,60 @@
 
 set -xu
 
+snapshot_files() {
+    SNAPSHOT_DIR=${1}
+    {{ range $dest, $source := .Values.anchor.files_to_copy }}
+    mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
+    cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
+    {{- end }}
+    {{ range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/scheduler/{{ $val.file }}"
+    {{- end }}
+    {{- end }}
+    # annotate the static manifest with the name of the creating anchor pod
+    sed -i "/created-by: /s/ANCHOR_POD/${POD_NAME}/" "${SNAPSHOT_DIR}{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-scheduler.yaml"
+}
+
 compare_copy_files() {
-    {{- range .Values.anchor.files_to_copy }}
+    SNAPSHOT_DIR=${1}
-    if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
+    {{ range $dest, $source := .Values.anchor.files_to_copy }}
-        mkdir -p $(dirname /host{{ .dest }})
+    SRC="${SNAPSHOT_DIR}{{ $dest }}"
-        cp {{ .source }} /host{{ .dest }}
+    DEST="/host{{ $dest }}"
-        chmod go-rwx /host{{ .dest }}
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
     fi
+    {{- end}}
+    {{ range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    SRC="${SNAPSHOT_DIR}/etc/kubernetes/scheduler/{{ $val.file }}"
+    DEST="/host/etc/kubernetes/scheduler/{{ $val.file }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
+    fi
+    {{- end }}
     {{- end }}
 }
 
 cleanup() {
-    {{- range .Values.anchor.files_to_copy }}
+    {{- range $dest, $source := .Values.anchor.files_to_copy }}
-    rm -f /host{{ .dest }}
+    rm -f "/host{{ $dest }}"
+    {{- end }}
+    {{  range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    rm -f "/host/etc/kubernetes/scheduler/{{ $val.file }}"
+    {{- end }}
     {{- end }}
 }
 
+SNAPSHOT_DIR=$(mktemp -d)
+
+snapshot_files "${SNAPSHOT_DIR}"
+
 while true; do
     if [ -e /tmp/stop ]; then
         echo Stopping
@@ -44,7 +82,7 @@ while true; do
 
     # Compare and replace files on Genesis host if needed
     # Copy files to other master nodes
-    compare_copy_files
+    compare_copy_files "${SNAPSHOT_DIR}"
 
     sleep {{ .Values.anchor.period }}
 done

charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl

@@ -25,6 +25,7 @@ metadata:
     {{ .Values.service.name }}-service: enabled
 {{ tuple $envAll "kubernetes" "scheduler" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
   annotations:
+    created-by: ANCHOR_POD
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
 {{ dict "envAll" $envAll "podName" "scheduler" "containerNames" (list "scheduler") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }}
 spec:

charts/scheduler/templates/sched-anchor.yaml

@@ -56,6 +56,15 @@ spec:
         - name: anchor
           image: {{ .Values.images.tags.anchor }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
+          env:
+          - name: MANIFEST_PATH
+            value: /host{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-scheduler.yaml
+          - name: ETC_PATH
+            value: /host{{ .Values.scheduler.host_etc_path }}
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
 {{ tuple $envAll $envAll.Values.pod.resources.anchor_daemonset | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "scheduler" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
@@ -67,11 +76,19 @@ spec:
                   - /tmp/bin/pre_stop
 
           readinessProbe:
-            httpGet:
+            exec:
-              host: 127.0.0.1
+              command:
-              path: /healthz
+              - /bin/bash
-              port: {{ .Values.network.kubernetes_scheduler.port }}
+              - -c
-              scheme: HTTPS
+              - |-
+                grep -q "created-by: ${POD_NAME}" "${MANIFEST_PATH}" || exit 1
+                [ "$(curl -k -s -S -o /dev/null \
+                    --cert "/host{{ .Values.scheduler.host_etc_path }}/scheduler.pem" \
+                    --key "/host{{ .Values.scheduler.host_etc_path }}/scheduler-key.pem" \
+                    --cacert "/host{{ .Values.scheduler.host_etc_path }}/cluster-ca.pem" \
+                    "https://localhost:{{ .Values.network.kubernetes_scheduler.port }}/healthz" \
+                    -w "%{http_code}")" = "200" ]
+                exit $?
             initialDelaySeconds: 10
             periodSeconds: 5
             timeoutSeconds: 5

charts/scheduler/values.yaml

@@ -8,16 +8,11 @@ anchor:
   period: 15
   termination_grace_period: 3600
   files_to_copy:
-    - source: /configmap/cluster-ca.pem
+    /etc/kubernetes/scheduler/cluster-ca.pem: /configmap/cluster-ca.pem
-      dest: /etc/kubernetes/scheduler/cluster-ca.pem
+    /etc/kubernetes/scheduler/scheduler.pem: /configmap/scheduler.pem
-    - source: /configmap/scheduler.pem
+    /etc/kubernetes/scheduler/kubeconfig.yaml: /configmap/kubeconfig.yaml
-      dest: /etc/kubernetes/scheduler/scheduler.pem
+    /etc/kubernetes/scheduler/scheduler-key.pem: /secret/scheduler-key.pem
-    - source: /configmap/kubeconfig.yaml
+    /etc/kubernetes/manifests/kubernetes-scheduler.yaml: /configmap/kubernetes-scheduler.yaml
-      dest: /etc/kubernetes/scheduler/kubeconfig.yaml
-    - source: /secret/scheduler-key.pem
-      dest: /etc/kubernetes/scheduler/scheduler-key.pem
-    - source: /configmap/kubernetes-scheduler.yaml
-      dest: /etc/kubernetes/manifests/kubernetes-scheduler.yaml
 
 labels:
   scheduler: