airship
/
promenade
Code Issues Proposed changes

Adjusting daemonset anchor readiness check 

To avoid pods cycling too quickly by checking if manifest
was created by daemonset and the component on the same host
is ready

Change-Id: I7f9b35e222ef5934fca71f30fdf9941caa60ccd7
This commit is contained in:
SPEARS, DUSTIN (ds443n) 2023-03-30 13:32:26 -04:00
parent 2125b61b57
commit 5f62088d01
10 changed files with 148 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
charts/controller_manager/Chart.yaml
View File

@ -15,4 +15,4 @@
apiVersion: v1
description: A chart for Kubernetes controller-manager
name: controller_manager
version: 0.1.1
version: 0.1.2

60
charts/controller_manager/templates/bin/_anchor.tpl
View File

@ -15,24 +15,60 @@
set -xu
compare_copy_files() {
snapshot_files() {
SNAPSHOT_DIR=${1}
{{ range $dest, $source := .Values.anchor.files_to_copy }}
mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
{{- end }}
{{ range $key, $val := .Values.conf }}
{{- if $val.file }}
cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/controller-manager/{{ $val.file }}"
{{- end }}
{{- end }}
# annotate the static manifest with the name of the creating anchor pod
sed -i "/created-by: /s/ANCHOR_POD/${POD_NAME}/" "${SNAPSHOT_DIR}{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-controller-manager.yaml"
}
{{range .Values.anchor.files_to_copy}}
if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
mkdir -p $(dirname /host{{ .dest }})
cp {{ .source }} /host{{ .dest }}
chmod go-rwx /host{{ .dest }}
compare_copy_files() {
SNAPSHOT_DIR=${1}
{{ range $dest, $source := .Values.anchor.files_to_copy }}
SRC="${SNAPSHOT_DIR}{{ $dest }}"
DEST="/host{{ $dest }}"
if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
mkdir -p $(dirname "${DEST}")
cp "${SRC}" "${DEST}"
chmod go-rwx "${DEST}"
fi
{{end}}
{{- end}}
{{ range $key, $val := .Values.conf }}
{{- if $val.file }}
SRC="${SNAPSHOT_DIR}/etc/kubernetes/controller-manager/{{ $val.file }}"
DEST="/host/etc/kubernetes/controller-manager/{{ $val.file }}"
if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
mkdir -p $(dirname "${DEST}")
cp "${SRC}" "${DEST}"
chmod go-rwx "${DEST}"
fi
{{- end }}
{{- end }}
}
cleanup() {
{{range .Values.anchor.files_to_copy}}
rm -f /host{{ .dest }}
{{end}}
{{- range $dest, $source := .Values.anchor.files_to_copy }}
rm -f "/host{{ $dest }}"
{{- end }}
{{ range $key, $val := .Values.conf }}
{{- if $val.file }}
rm -f "/host/etc/kubernetes/controller-manager/{{ $val.file }}"
{{- end }}
{{- end }}
}
SNAPSHOT_DIR=$(mktemp -d)
snapshot_files "${SNAPSHOT_DIR}"
while true; do
if [ -e /tmp/stop ]; then
@ -45,7 +81,7 @@ while true; do
# Compare and replace files on Genesis host if needed
# Copy files to other master nodes
compare_copy_files
compare_copy_files "${SNAPSHOT_DIR}"
sleep {{ .Values.anchor.period }}
done

23
charts/controller_manager/templates/daemonset.yaml
View File

@ -64,6 +64,10 @@ spec:
value: /host{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-controller-manager.yaml
- name: ETC_PATH
value: /host{{ .Values.controller_manager.host_etc_path }}
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
{{ tuple $envAll $envAll.Values.pod.resources.anchor_pod | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "kubernetes" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command:
@ -73,13 +77,20 @@ spec:
exec:
command:
- /tmp/bin/pre_stop
readinessProbe:
httpGet:
host: 127.0.0.1
path: /healthz
port: {{ .Values.network.kubernetes_controller_manager.port }}
scheme: HTTPS
exec:
command:
- /bin/bash
- -c
- |-
grep -q "created-by: ${POD_NAME}" "${MANIFEST_PATH}" || exit 1
[ "$(curl -k -s -S -o /dev/null \
--cert "/host{{ .Values.controller_manager.host_etc_path }}/controller-manager.pem" \
--key "/host{{ .Values.controller_manager.host_etc_path }}/controller-manager-key.pem" \
--cacert "/host{{ .Values.controller_manager.host_etc_path }}/cluster-ca.pem" \
"https://localhost:{{ .Values.network.kubernetes_controller_manager.port }}/healthz" \
-w "%{http_code}")" = "200" ]
exit $?
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 5

1
charts/controller_manager/templates/etc/_kubernetes-controller-manager.yaml.tpl
View File

@ -23,6 +23,7 @@ metadata:
{{ .Values.service.name }}-service: enabled
{{ tuple $envAll "kubernetes" "controller-manager" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
annotations:
created-by: ANCHOR_POD
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
{{ dict "envAll" $envAll "podName" "controller-manager" "containerNames" (list "controller-manager") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }}
spec:

18
charts/controller_manager/values.yaml
View File

@ -32,18 +32,12 @@ anchor:
manifest_path: /etc/kubernetes/manifests
period: 15
files_to_copy:
- source: /configmap/cluster-ca.pem
dest: /etc/kubernetes/controller-manager/cluster-ca.pem
- source: /configmap/controller-manager.pem
dest: /etc/kubernetes/controller-manager/controller-manager.pem
- source: /configmap/kubeconfig.yaml
dest: /etc/kubernetes/controller-manager/kubeconfig.yaml
- source: /secret/controller-manager-key.pem
dest: /etc/kubernetes/controller-manager/controller-manager-key.pem
- source: /secret/service-account.priv
dest: /etc/kubernetes/controller-manager/service-account.priv
- source: /configmap/kubernetes-controller-manager.yaml
dest: /etc/kubernetes/manifests/kubernetes-controller-manager.yaml
/etc/kubernetes/controller-manager/cluster-ca.pem: /configmap/cluster-ca.pem
/etc/kubernetes/controller-manager/controller-manager.pem: /configmap/controller-manager.pem
/etc/kubernetes/controller-manager/kubeconfig.yaml: /configmap/kubeconfig.yaml
/etc/kubernetes/controller-manager/controller-manager-key.pem: /secret/controller-manager-key.pem
/etc/kubernetes/controller-manager/service-account.priv: /secret/service-account.priv
/etc/kubernetes/manifests/kubernetes-controller-manager.yaml: /configmap/kubernetes-controller-manager.yaml
controller_manager:
host_etc_path: /etc/kubernetes/controller-manager

2
charts/scheduler/Chart.yaml
View File

@ -1,4 +1,4 @@
apiVersion: v1
description: A chart for Kubernetes scheduler.
name: scheduler
version: 0.1.1
version: 0.1.2

54
charts/scheduler/templates/bin/_anchor.tpl
View File

@ -17,22 +17,60 @@
set -xu
snapshot_files() {
SNAPSHOT_DIR=${1}
{{ range $dest, $source := .Values.anchor.files_to_copy }}
mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
{{- end }}
{{ range $key, $val := .Values.conf }}
{{- if $val.file }}
cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/scheduler/{{ $val.file }}"
{{- end }}
{{- end }}
# annotate the static manifest with the name of the creating anchor pod
sed -i "/created-by: /s/ANCHOR_POD/${POD_NAME}/" "${SNAPSHOT_DIR}{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-scheduler.yaml"
}
compare_copy_files() {
{{- range .Values.anchor.files_to_copy }}
if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
mkdir -p $(dirname /host{{ .dest }})
cp {{ .source }} /host{{ .dest }}
chmod go-rwx /host{{ .dest }}
SNAPSHOT_DIR=${1}
{{ range $dest, $source := .Values.anchor.files_to_copy }}
SRC="${SNAPSHOT_DIR}{{ $dest }}"
DEST="/host{{ $dest }}"
if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
mkdir -p $(dirname "${DEST}")
cp "${SRC}" "${DEST}"
chmod go-rwx "${DEST}"
fi
{{- end}}
{{ range $key, $val := .Values.conf }}
{{- if $val.file }}
SRC="${SNAPSHOT_DIR}/etc/kubernetes/scheduler/{{ $val.file }}"
DEST="/host/etc/kubernetes/scheduler/{{ $val.file }}"
if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
mkdir -p $(dirname "${DEST}")
cp "${SRC}" "${DEST}"
chmod go-rwx "${DEST}"
fi
{{- end }}
{{- end }}
}
cleanup() {
{{- range .Values.anchor.files_to_copy }}
rm -f /host{{ .dest }}
{{- range $dest, $source := .Values.anchor.files_to_copy }}
rm -f "/host{{ $dest }}"
{{- end }}
{{ range $key, $val := .Values.conf }}
{{- if $val.file }}
rm -f "/host/etc/kubernetes/scheduler/{{ $val.file }}"
{{- end }}
{{- end }}
}
SNAPSHOT_DIR=$(mktemp -d)
snapshot_files "${SNAPSHOT_DIR}"
while true; do
if [ -e /tmp/stop ]; then
echo Stopping
@ -44,7 +82,7 @@ while true; do
# Compare and replace files on Genesis host if needed
# Copy files to other master nodes
compare_copy_files
compare_copy_files "${SNAPSHOT_DIR}"
sleep {{ .Values.anchor.period }}
done

1
charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl
View File

@ -25,6 +25,7 @@ metadata:
{{ .Values.service.name }}-service: enabled
{{ tuple $envAll "kubernetes" "scheduler" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
annotations:
created-by: ANCHOR_POD
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
{{ dict "envAll" $envAll "podName" "scheduler" "containerNames" (list "scheduler") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }}
spec:

27
charts/scheduler/templates/sched-anchor.yaml
View File

@ -56,6 +56,15 @@ spec:
- name: anchor
image: {{ .Values.images.tags.anchor }}
imagePullPolicy: {{ .Values.images.pull_policy }}
env:
- name: MANIFEST_PATH
value: /host{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-scheduler.yaml
- name: ETC_PATH
value: /host{{ .Values.scheduler.host_etc_path }}
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
{{ tuple $envAll $envAll.Values.pod.resources.anchor_daemonset | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "scheduler" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command:
@ -67,11 +76,19 @@ spec:
- /tmp/bin/pre_stop
readinessProbe:
httpGet:
host: 127.0.0.1
path: /healthz
port: {{ .Values.network.kubernetes_scheduler.port }}
scheme: HTTPS
exec:
command:
- /bin/bash
- -c
- |-
grep -q "created-by: ${POD_NAME}" "${MANIFEST_PATH}" || exit 1
[ "$(curl -k -s -S -o /dev/null \
--cert "/host{{ .Values.scheduler.host_etc_path }}/scheduler.pem" \
--key "/host{{ .Values.scheduler.host_etc_path }}/scheduler-key.pem" \
--cacert "/host{{ .Values.scheduler.host_etc_path }}/cluster-ca.pem" \
"https://localhost:{{ .Values.network.kubernetes_scheduler.port }}/healthz" \
-w "%{http_code}")" = "200" ]
exit $?
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 5

15
charts/scheduler/values.yaml
View File

@ -8,16 +8,11 @@ anchor:
period: 15
termination_grace_period: 3600
files_to_copy:
- source: /configmap/cluster-ca.pem
dest: /etc/kubernetes/scheduler/cluster-ca.pem
- source: /configmap/scheduler.pem
dest: /etc/kubernetes/scheduler/scheduler.pem
- source: /configmap/kubeconfig.yaml
dest: /etc/kubernetes/scheduler/kubeconfig.yaml
- source: /secret/scheduler-key.pem
dest: /etc/kubernetes/scheduler/scheduler-key.pem
- source: /configmap/kubernetes-scheduler.yaml
dest: /etc/kubernetes/manifests/kubernetes-scheduler.yaml
/etc/kubernetes/scheduler/cluster-ca.pem: /configmap/cluster-ca.pem
/etc/kubernetes/scheduler/scheduler.pem: /configmap/scheduler.pem
/etc/kubernetes/scheduler/kubeconfig.yaml: /configmap/kubeconfig.yaml
/etc/kubernetes/scheduler/scheduler-key.pem: /secret/scheduler-key.pem
/etc/kubernetes/manifests/kubernetes-scheduler.yaml: /configmap/kubernetes-scheduler.yaml
labels:
scheduler: