From 08906262fd856b9fbf092fa11462a7ab591f52e7 Mon Sep 17 00:00:00 2001 From: Phil Sphicas Date: Mon, 18 Oct 2021 11:30:50 -0700 Subject: [PATCH] Update tolerations and priority classes * Give kube-proxy a blanket toleration * Replace scheduler.alpha.kubernetes.io/critical-pod annotation with priorityClassName: system-node-critical Change-Id: I810333913c09531eefa1ded014fe090d4cca7f7d --- charts/apiserver/templates/daemonset.yaml | 2 +- charts/controller_manager/templates/daemonset.yaml | 2 +- charts/etcd/templates/daemonset-anchor.yaml | 2 +- charts/etcd/templates/tests/test-etcd-health.yaml | 1 - charts/haproxy/templates/daemonset.yaml | 2 +- charts/proxy/templates/daemonset.yaml | 10 ++-------- charts/scheduler/templates/sched-anchor.yaml | 2 +- .../roles/common/etc/kubernetes/manifests/haproxy.yaml | 3 +-- .../etc/kubernetes/manifests/kubernetes-apiserver.yaml | 3 +-- .../manifests/kubernetes-controller-manager.yaml | 3 +-- .../etc/kubernetes/manifests/kubernetes-scheduler.yaml | 3 +-- 11 files changed, 11 insertions(+), 22 deletions(-) diff --git a/charts/apiserver/templates/daemonset.yaml b/charts/apiserver/templates/daemonset.yaml index e5024b39..cfae07d7 100644 --- a/charts/apiserver/templates/daemonset.yaml +++ b/charts/apiserver/templates/daemonset.yaml @@ -42,7 +42,6 @@ spec: {{ $labels | indent 8 }} annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} - scheduler.alpha.kubernetes.io/critical-pod: '' configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} {{ dict "envAll" $envAll "podName" "kubernetes_apiserver_anchor" "containerNames" (list "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} @@ -52,6 +51,7 @@ spec: {{ .Values.labels.kubernetes_apiserver.node_selector_key }}: {{ .Values.labels.kubernetes_apiserver.node_selector_value }} dnsPolicy: {{ .Values.anchor.dns_policy }} hostNetwork: true + priorityClassName: system-node-critical tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule diff --git a/charts/controller_manager/templates/daemonset.yaml b/charts/controller_manager/templates/daemonset.yaml index 5e67b4e3..27946803 100644 --- a/charts/controller_manager/templates/daemonset.yaml +++ b/charts/controller_manager/templates/daemonset.yaml @@ -39,7 +39,6 @@ spec: {{ $labels | indent 8 }} annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} - scheduler.alpha.kubernetes.io/critical-pod: '' configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} {{ dict "envAll" $envAll "podName" "kubernetes-controller-manager-anchor" "containerNames" (list "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} @@ -49,6 +48,7 @@ spec: {{ .Values.labels.controller_manager.node_selector_key }}: {{ .Values.labels.controller_manager.node_selector_value }} dnsPolicy: {{ .Values.anchor.dns_policy }} hostNetwork: true + priorityClassName: system-node-critical tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule diff --git a/charts/etcd/templates/daemonset-anchor.yaml b/charts/etcd/templates/daemonset-anchor.yaml index 1f74854e..bc832016 100644 --- a/charts/etcd/templates/daemonset-anchor.yaml +++ b/charts/etcd/templates/daemonset-anchor.yaml @@ -41,7 +41,6 @@ spec: annotations: {{ dict "envAll" $envAll "podName" "etcd-anchor" "containerNames" (list "etcdctl") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} - scheduler.alpha.kubernetes.io/critical-pod: '' {{- if .Values.manifests.configmap_bin }} configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} {{- end }} @@ -59,6 +58,7 @@ spec: {{- end }} nodeSelector: {{ .Values.labels.anchor.node_selector_key }}: {{ .Values.labels.anchor.node_selector_value }} + priorityClassName: system-node-critical tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule diff --git a/charts/etcd/templates/tests/test-etcd-health.yaml b/charts/etcd/templates/tests/test-etcd-health.yaml index cfcca716..d630f0dd 100644 --- a/charts/etcd/templates/tests/test-etcd-health.yaml +++ b/charts/etcd/templates/tests/test-etcd-health.yaml @@ -25,7 +25,6 @@ metadata: name: "{{ .Release.Name }}-etcd-test" annotations: "helm.sh/hook": "test" - scheduler.alpha.kubernetes.io/critical-pod: '' {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} {{ dict "envAll" $envAll "podName" "etcd-test" "containerNames" (list "etcd-test") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} labels: diff --git a/charts/haproxy/templates/daemonset.yaml b/charts/haproxy/templates/daemonset.yaml index afe8ffac..1881eeb7 100644 --- a/charts/haproxy/templates/daemonset.yaml +++ b/charts/haproxy/templates/daemonset.yaml @@ -36,10 +36,10 @@ spec: {{ $labels | indent 8 }} annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} - scheduler.alpha.kubernetes.io/critical-pod: '' {{ dict "envAll" $envAll "podName" "haproxy-anchor" "containerNames" (list "haproxy-perms" "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} spec: {{ dict "envAll" $envAll "application" "haproxy_anchor" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + priorityClassName: system-node-critical tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule diff --git a/charts/proxy/templates/daemonset.yaml b/charts/proxy/templates/daemonset.yaml index ac3b9dc9..97b18162 100644 --- a/charts/proxy/templates/daemonset.yaml +++ b/charts/proxy/templates/daemonset.yaml @@ -45,7 +45,6 @@ spec: annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} {{ dict "envAll" $envAll "podName" "kubernetes-proxy" "containerNames" (list "proxy") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} - scheduler.alpha.kubernetes.io/critical-pod: '' {{- if .Values.manifests.configmap_proxy }} configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} {{- end }} @@ -54,14 +53,9 @@ spec: hostNetwork: true shareProcessNamespace: true dnsPolicy: Default + priorityClassName: system-node-critical tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - - key: CriticalAddonsOnly - operator: Exists - - key: node.kubernetes.io/not-ready - operator: Exists - effect: NoSchedule + - operator: Exists containers: - name: proxy image: {{ .Values.images.tags.proxy }} diff --git a/charts/scheduler/templates/sched-anchor.yaml b/charts/scheduler/templates/sched-anchor.yaml index ba4d62d1..3a8e91b1 100644 --- a/charts/scheduler/templates/sched-anchor.yaml +++ b/charts/scheduler/templates/sched-anchor.yaml @@ -34,7 +34,6 @@ spec: metadata: annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} - scheduler.alpha.kubernetes.io/critical-pod: '' configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} {{ dict "envAll" $envAll "podName" "scheduler" "containerNames" (list "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} @@ -46,6 +45,7 @@ spec: dnsPolicy: {{ .Values.anchor.dns_policy }} nodeSelector: {{ .Values.labels.scheduler.node_selector_key }}: {{ .Values.labels.scheduler.node_selector_value }} + priorityClassName: system-node-critical tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule diff --git a/promenade/templates/roles/common/etc/kubernetes/manifests/haproxy.yaml b/promenade/templates/roles/common/etc/kubernetes/manifests/haproxy.yaml index 4430ae74..a113893c 100644 --- a/promenade/templates/roles/common/etc/kubernetes/manifests/haproxy.yaml +++ b/promenade/templates/roles/common/etc/kubernetes/manifests/haproxy.yaml @@ -4,10 +4,9 @@ kind: Pod metadata: name: haproxy namespace: kube-system - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: hostNetwork: true + priorityClassName: system-node-critical containers: - name: haproxy image: {{ config['HostSystem:images.haproxy'] }} diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml index 18544268..72923dd4 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml @@ -8,10 +8,9 @@ metadata: application: kubernetes component: apiserver kubernetes-apiserver-service: enabled - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: hostNetwork: true + priorityClassName: system-node-critical containers: - name: kube-apiserver image: {{ config['Genesis:images.kubernetes.apiserver'] }} diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-controller-manager.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-controller-manager.yaml index 97716441..24b0fe47 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-controller-manager.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-controller-manager.yaml @@ -8,10 +8,9 @@ metadata: tier: control-plane application: kubernetes component: kube-controller-manager - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: hostNetwork: true + priorityClassName: system-node-critical containers: - name: kube-controller-manager image: {{ config['Genesis:images.kubernetes.controller-manager'] }} diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-scheduler.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-scheduler.yaml index c771bdac..8721557e 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-scheduler.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-scheduler.yaml @@ -8,10 +8,9 @@ metadata: tier: control-plane application: kubernetes component: kube-scheduler - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: hostNetwork: true + priorityClassName: system-node-critical containers: - name: kube-scheduler image: {{ config['Genesis:images.kubernetes.scheduler'] }}