From 178193be8443eac9204adfff49f88efa12abfecb Mon Sep 17 00:00:00 2001 From: Matt McEuen Date: Wed, 17 Oct 2018 18:22:04 -0500 Subject: [PATCH] Add EventRateLimit admission controller Add the EventRateLimit admission controller, to allow operators to define rate limits for the k8s API server at the server, namespace, or user account level. This also * cleans up some of the parameters passed into the API server * replaces the deprecated --admission-control parameter * applies --repair-malformed-updates consistently, incl examples * removes unused batch/v2alpha1 runtime config * https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/ * removes duplicate --service-cluster-ip-range setting This PS adds EventRateLimits to the bootstrap and anchor API servers; future work will need to add it to the Keystone Webhook API server. Change-Id: I32a2d4add880e50f470e4cb0687e20d16e6e926d --- charts/apiserver/templates/configmap-etc.yaml | 20 ++++++++++++++ .../etc/_kubernetes-apiserver.yaml.tpl | 1 + charts/apiserver/values.yaml | 27 ++++++++++++++++--- examples/basic/Genesis.yaml | 2 +- examples/basic/armada-resources.yaml | 2 +- .../etc/genesis/apiserver/acconfig.yaml | 6 +++++ .../etc/genesis/apiserver/eventconfig.yaml | 7 +++++ .../manifests/bootstrap-armada.yaml | 5 +--- .../manifests/kubernetes-apiserver.yaml | 5 +--- 9 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml create mode 100644 promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml diff --git a/charts/apiserver/templates/configmap-etc.yaml b/charts/apiserver/templates/configmap-etc.yaml index 725f52ac..75a22eac 100644 --- a/charts/apiserver/templates/configmap-etc.yaml +++ b/charts/apiserver/templates/configmap-etc.yaml @@ -17,6 +17,21 @@ limitations under the License. {{- if .Values.manifests.configmap_etc }} {{- $envAll := . }} +{{/* This slightly involved merge of AC config files into the anchor + files uses HTK merge, as straighforward appends result in duplicates. */}} +{{- $_ := set .Values "_ac_files_to_copy" list }} +{{- range $key, $val := .Values.conf.admission_controllers }} + {{- $source := printf "/tmp/etc/%s" $key }} + {{- $dest := printf "/etc/kubernetes/apiserver/%s" $key }} + {{- $file_to_copy := dict "source" $source "dest" $dest }} + {{- $ac_files_to_copy := append $.Values._ac_files_to_copy $file_to_copy }} + {{- $_ := set $.Values "_ac_files_to_copy" $ac_files_to_copy }} +{{- end }} +{{ $all_files_to_copy := dict }} +{{ $_ := set $all_files_to_copy "values" (tuple .Values.anchor.files_to_copy .Values._ac_files_to_copy) }} +{{ $_ := $all_files_to_copy | include "helm-toolkit.utils.merge" }} +{{ $_ := set .Values.anchor "files_to_copy" $all_files_to_copy.result }} + --- apiVersion: v1 kind: ConfigMap @@ -27,4 +42,9 @@ data: {{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} kubeconfig.yaml: |+ {{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{/* Dynamically add config files for admission controllers */}} +{{ range $key, $val := .Values.conf.admission_controllers }} + {{ $key }}: |+ +{{ toYaml $val | indent 4 }} +{{ end }} {{- end }} diff --git a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl index 0d2f36da..70fbd407 100644 --- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl +++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl @@ -62,6 +62,7 @@ spec: - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem - --allow-privileged=true - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub + - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml ports: - containerPort: {{ .Values.network.kubernetes_apiserver.port }} diff --git a/charts/apiserver/values.yaml b/charts/apiserver/values.yaml index f4bc653f..b1dc0c6b 100644 --- a/charts/apiserver/values.yaml +++ b/charts/apiserver/values.yaml @@ -55,20 +55,41 @@ anchor: dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml - source: /tmp/etc/kubeconfig.yaml dest: /etc/kubernetes/apiserver/kubeconfig.yaml + # Note: config files for admission controllers are added to this dynamically command_prefix: - /apiserver - --authorization-mode=Node,RBAC - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds - - --apiserver-count=3 + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit - --service-cluster-ip-range=10.96.0.0/16 - - --v=5 + - --endpoint-reconciler-type=lease + # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 + - --repair-malformed-updates=false apiserver: host_etc_path: /etc/kubernetes/apiserver etcd: endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local +conf: + # Admission controllers config files are generated dynamically based on the + # config below, as they they are specific to particular ACs that may be + # configured by the operator (or added by k8s in the future). + admission_controllers: + eventconfig.yaml: + kind: Configuration + apiVersion: eventratelimit.admission.k8s.io/v1alpha1 + limits: + - type: Server + qps: 100 + burst: 1000 + acconfig.yaml: + kind: AdmissionConfiguration + apiVersion: apiserver.k8s.io/v1alpha1 + plugins: + - name: EventRateLimit + path: eventconfig.yaml + network: kubernetes_apiserver: ingress: diff --git a/examples/basic/Genesis.yaml b/examples/basic/Genesis.yaml index 7bade7d2..3f11c827 100644 --- a/examples/basic/Genesis.yaml +++ b/examples/basic/Genesis.yaml @@ -14,7 +14,7 @@ data: command_prefix: - /apiserver - --authorization-mode=Node,RBAC - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit - --service-cluster-ip-range=10.96.0.0/16 - --endpoint-reconciler-type=lease # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index 8b49feac..d5330f98 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -743,7 +743,7 @@ data: command_prefix: - /apiserver - --authorization-mode=Node,RBAC - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit - --service-cluster-ip-range=10.96.0.0/16 - --endpoint-reconciler-type=lease # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml new file mode 100644 index 00000000..c792a8b4 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml @@ -0,0 +1,6 @@ +--- +kind: AdmissionConfiguration +apiVersion: apiserver.k8s.io/v1alpha1 +plugins: +- name: EventRateLimit + path: eventconfig.yaml \ No newline at end of file diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml new file mode 100644 index 00000000..ae789689 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml @@ -0,0 +1,7 @@ +--- +kind: Configuration +apiVersion: eventratelimit.admission.k8s.io/v1alpha1 +limits: +- type: Server + qps: 100 + burst: 1000 \ No newline at end of file diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml index adfa41ef..3e6e48b1 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml @@ -122,8 +122,6 @@ spec: - "{{ argument }}" {%- endfor %} - --advertise-address={{ config['Genesis:ip'] }} - - --authorization-mode=Node,RBAC - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds - --anonymous-auth=false - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem @@ -132,15 +130,14 @@ spec: - --insecure-port=8080 - --secure-port=6444 - --bind-address=0.0.0.0 - - --runtime-config=batch/v2alpha1=true - --allow-privileged=true - --etcd-servers=https://localhost:12379 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem - - --service-cluster-ip-range={{ config['KubernetesNetwork:kubernetes.service_cidr'] }} - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub + - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem env: diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml index ef27e8b5..606f0f3e 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml @@ -20,8 +20,6 @@ spec: - "{{ argument }}" {%- endfor %} - --advertise-address={{ config['Genesis:ip'] }} - - --authorization-mode=Node,RBAC - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds - --anonymous-auth=false - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem @@ -30,15 +28,14 @@ spec: - --insecure-port=0 - --bind-address=0.0.0.0 - --secure-port=6443 - - --runtime-config=batch/v2alpha1=true - --allow-privileged=true - --etcd-servers=https://localhost:2379 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem - - --service-cluster-ip-range={{ config['KubernetesNetwork:kubernetes.service_cidr'] }} - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub + - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem volumeMounts: