diff --git a/charts/controller_manager/Chart.yaml b/charts/controller_manager/Chart.yaml index ae00f1ce..322afc8c 100644 --- a/charts/controller_manager/Chart.yaml +++ b/charts/controller_manager/Chart.yaml @@ -15,4 +15,4 @@ apiVersion: v1 description: A chart for Kubernetes controller-manager name: controller_manager -version: 0.1.1 +version: 0.1.2 diff --git a/charts/controller_manager/templates/bin/_anchor.tpl b/charts/controller_manager/templates/bin/_anchor.tpl index 20b76292..c27a5d30 100644 --- a/charts/controller_manager/templates/bin/_anchor.tpl +++ b/charts/controller_manager/templates/bin/_anchor.tpl @@ -15,24 +15,60 @@ set -xu -compare_copy_files() { +snapshot_files() { + SNAPSHOT_DIR=${1} + {{ range $dest, $source := .Values.anchor.files_to_copy }} + mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}") + cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}" + {{- end }} + {{ range $key, $val := .Values.conf }} + {{- if $val.file }} + cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/controller-manager/{{ $val.file }}" + {{- end }} + {{- end }} + # annotate the static manifest with the name of the creating anchor pod + sed -i "/created-by: /s/ANCHOR_POD/${POD_NAME}/" "${SNAPSHOT_DIR}{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-controller-manager.yaml" +} - {{range .Values.anchor.files_to_copy}} - if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then - mkdir -p $(dirname /host{{ .dest }}) - cp {{ .source }} /host{{ .dest }} - chmod go-rwx /host{{ .dest }} +compare_copy_files() { + SNAPSHOT_DIR=${1} + {{ range $dest, $source := .Values.anchor.files_to_copy }} + SRC="${SNAPSHOT_DIR}{{ $dest }}" + DEST="/host{{ $dest }}" + if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then + mkdir -p $(dirname "${DEST}") + cp "${SRC}" "${DEST}" + chmod go-rwx "${DEST}" fi - {{end}} + {{- end}} + {{ range $key, $val := .Values.conf }} + {{- if $val.file }} + SRC="${SNAPSHOT_DIR}/etc/kubernetes/controller-manager/{{ $val.file }}" + DEST="/host/etc/kubernetes/controller-manager/{{ $val.file }}" + if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then + mkdir -p $(dirname "${DEST}") + cp "${SRC}" "${DEST}" + chmod go-rwx "${DEST}" + fi + {{- end }} + {{- end }} } cleanup() { - - {{range .Values.anchor.files_to_copy}} - rm -f /host{{ .dest }} - {{end}} + {{- range $dest, $source := .Values.anchor.files_to_copy }} + rm -f "/host{{ $dest }}" + {{- end }} + {{ range $key, $val := .Values.conf }} + {{- if $val.file }} + rm -f "/host/etc/kubernetes/controller-manager/{{ $val.file }}" + {{- end }} + {{- end }} } +SNAPSHOT_DIR=$(mktemp -d) + +snapshot_files "${SNAPSHOT_DIR}" + while true; do if [ -e /tmp/stop ]; then @@ -45,7 +81,7 @@ while true; do # Compare and replace files on Genesis host if needed # Copy files to other master nodes - compare_copy_files + compare_copy_files "${SNAPSHOT_DIR}" sleep {{ .Values.anchor.period }} done diff --git a/charts/controller_manager/templates/daemonset.yaml b/charts/controller_manager/templates/daemonset.yaml index d409214d..5c231640 100644 --- a/charts/controller_manager/templates/daemonset.yaml +++ b/charts/controller_manager/templates/daemonset.yaml @@ -64,6 +64,10 @@ spec: value: /host{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-controller-manager.yaml - name: ETC_PATH value: /host{{ .Values.controller_manager.host_etc_path }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name {{ tuple $envAll $envAll.Values.pod.resources.anchor_pod | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ dict "envAll" $envAll "application" "kubernetes" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} command: @@ -73,13 +77,20 @@ spec: exec: command: - /tmp/bin/pre_stop - readinessProbe: - httpGet: - host: 127.0.0.1 - path: /healthz - port: {{ .Values.network.kubernetes_controller_manager.port }} - scheme: HTTPS + exec: + command: + - /bin/bash + - -c + - |- + grep -q "created-by: ${POD_NAME}" "${MANIFEST_PATH}" || exit 1 + [ "$(curl -k -s -S -o /dev/null \ + --cert "/host{{ .Values.controller_manager.host_etc_path }}/controller-manager.pem" \ + --key "/host{{ .Values.controller_manager.host_etc_path }}/controller-manager-key.pem" \ + --cacert "/host{{ .Values.controller_manager.host_etc_path }}/cluster-ca.pem" \ + "https://localhost:{{ .Values.network.kubernetes_controller_manager.port }}/healthz" \ + -w "%{http_code}")" = "200" ] + exit $? initialDelaySeconds: 10 periodSeconds: 5 timeoutSeconds: 5 diff --git a/charts/controller_manager/templates/etc/_kubernetes-controller-manager.yaml.tpl b/charts/controller_manager/templates/etc/_kubernetes-controller-manager.yaml.tpl index 50d265d5..564e116f 100644 --- a/charts/controller_manager/templates/etc/_kubernetes-controller-manager.yaml.tpl +++ b/charts/controller_manager/templates/etc/_kubernetes-controller-manager.yaml.tpl @@ -23,6 +23,7 @@ metadata: {{ .Values.service.name }}-service: enabled {{ tuple $envAll "kubernetes" "controller-manager" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} annotations: + created-by: ANCHOR_POD {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} {{ dict "envAll" $envAll "podName" "controller-manager" "containerNames" (list "controller-manager") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} spec: diff --git a/charts/controller_manager/values.yaml b/charts/controller_manager/values.yaml index c0eb926a..bdcd72fe 100644 --- a/charts/controller_manager/values.yaml +++ b/charts/controller_manager/values.yaml @@ -32,18 +32,12 @@ anchor: manifest_path: /etc/kubernetes/manifests period: 15 files_to_copy: - - source: /configmap/cluster-ca.pem - dest: /etc/kubernetes/controller-manager/cluster-ca.pem - - source: /configmap/controller-manager.pem - dest: /etc/kubernetes/controller-manager/controller-manager.pem - - source: /configmap/kubeconfig.yaml - dest: /etc/kubernetes/controller-manager/kubeconfig.yaml - - source: /secret/controller-manager-key.pem - dest: /etc/kubernetes/controller-manager/controller-manager-key.pem - - source: /secret/service-account.priv - dest: /etc/kubernetes/controller-manager/service-account.priv - - source: /configmap/kubernetes-controller-manager.yaml - dest: /etc/kubernetes/manifests/kubernetes-controller-manager.yaml + /etc/kubernetes/controller-manager/cluster-ca.pem: /configmap/cluster-ca.pem + /etc/kubernetes/controller-manager/controller-manager.pem: /configmap/controller-manager.pem + /etc/kubernetes/controller-manager/kubeconfig.yaml: /configmap/kubeconfig.yaml + /etc/kubernetes/controller-manager/controller-manager-key.pem: /secret/controller-manager-key.pem + /etc/kubernetes/controller-manager/service-account.priv: /secret/service-account.priv + /etc/kubernetes/manifests/kubernetes-controller-manager.yaml: /configmap/kubernetes-controller-manager.yaml controller_manager: host_etc_path: /etc/kubernetes/controller-manager diff --git a/charts/scheduler/Chart.yaml b/charts/scheduler/Chart.yaml index c470c897..347252cd 100644 --- a/charts/scheduler/Chart.yaml +++ b/charts/scheduler/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: A chart for Kubernetes scheduler. name: scheduler -version: 0.1.1 +version: 0.1.2 diff --git a/charts/scheduler/templates/bin/_anchor.tpl b/charts/scheduler/templates/bin/_anchor.tpl index 9d8a2da0..1f0fe197 100644 --- a/charts/scheduler/templates/bin/_anchor.tpl +++ b/charts/scheduler/templates/bin/_anchor.tpl @@ -17,22 +17,60 @@ set -xu +snapshot_files() { + SNAPSHOT_DIR=${1} + {{ range $dest, $source := .Values.anchor.files_to_copy }} + mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}") + cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}" + {{- end }} + {{ range $key, $val := .Values.conf }} + {{- if $val.file }} + cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/scheduler/{{ $val.file }}" + {{- end }} + {{- end }} + # annotate the static manifest with the name of the creating anchor pod + sed -i "/created-by: /s/ANCHOR_POD/${POD_NAME}/" "${SNAPSHOT_DIR}{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-scheduler.yaml" +} + compare_copy_files() { - {{- range .Values.anchor.files_to_copy }} - if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then - mkdir -p $(dirname /host{{ .dest }}) - cp {{ .source }} /host{{ .dest }} - chmod go-rwx /host{{ .dest }} + SNAPSHOT_DIR=${1} + {{ range $dest, $source := .Values.anchor.files_to_copy }} + SRC="${SNAPSHOT_DIR}{{ $dest }}" + DEST="/host{{ $dest }}" + if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then + mkdir -p $(dirname "${DEST}") + cp "${SRC}" "${DEST}" + chmod go-rwx "${DEST}" fi + {{- end}} + {{ range $key, $val := .Values.conf }} + {{- if $val.file }} + SRC="${SNAPSHOT_DIR}/etc/kubernetes/scheduler/{{ $val.file }}" + DEST="/host/etc/kubernetes/scheduler/{{ $val.file }}" + if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then + mkdir -p $(dirname "${DEST}") + cp "${SRC}" "${DEST}" + chmod go-rwx "${DEST}" + fi + {{- end }} {{- end }} } cleanup() { - {{- range .Values.anchor.files_to_copy }} - rm -f /host{{ .dest }} + {{- range $dest, $source := .Values.anchor.files_to_copy }} + rm -f "/host{{ $dest }}" + {{- end }} + {{ range $key, $val := .Values.conf }} + {{- if $val.file }} + rm -f "/host/etc/kubernetes/scheduler/{{ $val.file }}" + {{- end }} {{- end }} } +SNAPSHOT_DIR=$(mktemp -d) + +snapshot_files "${SNAPSHOT_DIR}" + while true; do if [ -e /tmp/stop ]; then echo Stopping @@ -44,7 +82,7 @@ while true; do # Compare and replace files on Genesis host if needed # Copy files to other master nodes - compare_copy_files + compare_copy_files "${SNAPSHOT_DIR}" sleep {{ .Values.anchor.period }} done diff --git a/charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl b/charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl index c6108338..45803c34 100644 --- a/charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl +++ b/charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl @@ -25,6 +25,7 @@ metadata: {{ .Values.service.name }}-service: enabled {{ tuple $envAll "kubernetes" "scheduler" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} annotations: + created-by: ANCHOR_POD {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} {{ dict "envAll" $envAll "podName" "scheduler" "containerNames" (list "scheduler") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} spec: diff --git a/charts/scheduler/templates/sched-anchor.yaml b/charts/scheduler/templates/sched-anchor.yaml index ce5df100..82d5d61c 100644 --- a/charts/scheduler/templates/sched-anchor.yaml +++ b/charts/scheduler/templates/sched-anchor.yaml @@ -56,6 +56,15 @@ spec: - name: anchor image: {{ .Values.images.tags.anchor }} imagePullPolicy: {{ .Values.images.pull_policy }} + env: + - name: MANIFEST_PATH + value: /host{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-scheduler.yaml + - name: ETC_PATH + value: /host{{ .Values.scheduler.host_etc_path }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name {{ tuple $envAll $envAll.Values.pod.resources.anchor_daemonset | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ dict "envAll" $envAll "application" "scheduler" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} command: @@ -67,11 +76,19 @@ spec: - /tmp/bin/pre_stop readinessProbe: - httpGet: - host: 127.0.0.1 - path: /healthz - port: {{ .Values.network.kubernetes_scheduler.port }} - scheme: HTTPS + exec: + command: + - /bin/bash + - -c + - |- + grep -q "created-by: ${POD_NAME}" "${MANIFEST_PATH}" || exit 1 + [ "$(curl -k -s -S -o /dev/null \ + --cert "/host{{ .Values.scheduler.host_etc_path }}/scheduler.pem" \ + --key "/host{{ .Values.scheduler.host_etc_path }}/scheduler-key.pem" \ + --cacert "/host{{ .Values.scheduler.host_etc_path }}/cluster-ca.pem" \ + "https://localhost:{{ .Values.network.kubernetes_scheduler.port }}/healthz" \ + -w "%{http_code}")" = "200" ] + exit $? initialDelaySeconds: 10 periodSeconds: 5 timeoutSeconds: 5 diff --git a/charts/scheduler/values.yaml b/charts/scheduler/values.yaml index d82d1419..6025686c 100644 --- a/charts/scheduler/values.yaml +++ b/charts/scheduler/values.yaml @@ -8,16 +8,11 @@ anchor: period: 15 termination_grace_period: 3600 files_to_copy: - - source: /configmap/cluster-ca.pem - dest: /etc/kubernetes/scheduler/cluster-ca.pem - - source: /configmap/scheduler.pem - dest: /etc/kubernetes/scheduler/scheduler.pem - - source: /configmap/kubeconfig.yaml - dest: /etc/kubernetes/scheduler/kubeconfig.yaml - - source: /secret/scheduler-key.pem - dest: /etc/kubernetes/scheduler/scheduler-key.pem - - source: /configmap/kubernetes-scheduler.yaml - dest: /etc/kubernetes/manifests/kubernetes-scheduler.yaml + /etc/kubernetes/scheduler/cluster-ca.pem: /configmap/cluster-ca.pem + /etc/kubernetes/scheduler/scheduler.pem: /configmap/scheduler.pem + /etc/kubernetes/scheduler/kubeconfig.yaml: /configmap/kubeconfig.yaml + /etc/kubernetes/scheduler/scheduler-key.pem: /secret/scheduler-key.pem + /etc/kubernetes/manifests/kubernetes-scheduler.yaml: /configmap/kubernetes-scheduler.yaml labels: scheduler: