diff --git a/charts/apiserver/templates/configmap-etc.yaml b/charts/apiserver/templates/configmap-etc.yaml index 725f52ac..75a22eac 100644 --- a/charts/apiserver/templates/configmap-etc.yaml +++ b/charts/apiserver/templates/configmap-etc.yaml @@ -17,6 +17,21 @@ limitations under the License. {{- if .Values.manifests.configmap_etc }} {{- $envAll := . }} +{{/* This slightly involved merge of AC config files into the anchor + files uses HTK merge, as straighforward appends result in duplicates. */}} +{{- $_ := set .Values "_ac_files_to_copy" list }} +{{- range $key, $val := .Values.conf.admission_controllers }} + {{- $source := printf "/tmp/etc/%s" $key }} + {{- $dest := printf "/etc/kubernetes/apiserver/%s" $key }} + {{- $file_to_copy := dict "source" $source "dest" $dest }} + {{- $ac_files_to_copy := append $.Values._ac_files_to_copy $file_to_copy }} + {{- $_ := set $.Values "_ac_files_to_copy" $ac_files_to_copy }} +{{- end }} +{{ $all_files_to_copy := dict }} +{{ $_ := set $all_files_to_copy "values" (tuple .Values.anchor.files_to_copy .Values._ac_files_to_copy) }} +{{ $_ := $all_files_to_copy | include "helm-toolkit.utils.merge" }} +{{ $_ := set .Values.anchor "files_to_copy" $all_files_to_copy.result }} + --- apiVersion: v1 kind: ConfigMap @@ -27,4 +42,9 @@ data: {{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} kubeconfig.yaml: |+ {{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{/* Dynamically add config files for admission controllers */}} +{{ range $key, $val := .Values.conf.admission_controllers }} + {{ $key }}: |+ +{{ toYaml $val | indent 4 }} +{{ end }} {{- end }} diff --git a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl index 1d43331a..1ef6fac2 100644 --- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl +++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl @@ -63,6 +63,7 @@ spec: - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem - --allow-privileged=true - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub + - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml ports: - containerPort: {{ .Values.network.kubernetes_apiserver.port }} diff --git a/charts/apiserver/values.yaml b/charts/apiserver/values.yaml index f4bc653f..b1dc0c6b 100644 --- a/charts/apiserver/values.yaml +++ b/charts/apiserver/values.yaml @@ -55,20 +55,41 @@ anchor: dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml - source: /tmp/etc/kubeconfig.yaml dest: /etc/kubernetes/apiserver/kubeconfig.yaml + # Note: config files for admission controllers are added to this dynamically command_prefix: - /apiserver - --authorization-mode=Node,RBAC - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds - - --apiserver-count=3 + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit - --service-cluster-ip-range=10.96.0.0/16 - - --v=5 + - --endpoint-reconciler-type=lease + # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 + - --repair-malformed-updates=false apiserver: host_etc_path: /etc/kubernetes/apiserver etcd: endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local +conf: + # Admission controllers config files are generated dynamically based on the + # config below, as they they are specific to particular ACs that may be + # configured by the operator (or added by k8s in the future). + admission_controllers: + eventconfig.yaml: + kind: Configuration + apiVersion: eventratelimit.admission.k8s.io/v1alpha1 + limits: + - type: Server + qps: 100 + burst: 1000 + acconfig.yaml: + kind: AdmissionConfiguration + apiVersion: apiserver.k8s.io/v1alpha1 + plugins: + - name: EventRateLimit + path: eventconfig.yaml + network: kubernetes_apiserver: ingress: diff --git a/examples/basic/Genesis.yaml b/examples/basic/Genesis.yaml index 9079fa9f..9d64e052 100644 --- a/examples/basic/Genesis.yaml +++ b/examples/basic/Genesis.yaml @@ -14,7 +14,7 @@ data: command_prefix: - /apiserver - --authorization-mode=Node,RBAC - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit - --service-cluster-ip-range=10.96.0.0/16 - --endpoint-reconciler-type=lease - --feature-gates=PodShareProcessNamespace=true diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index f39b7a8d..56fabd25 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -721,7 +721,7 @@ data: command_prefix: - /apiserver - --authorization-mode=Node,RBAC - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit - --service-cluster-ip-range=10.96.0.0/16 - --endpoint-reconciler-type=lease - --feature-gates=PodShareProcessNamespace=true diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml new file mode 100644 index 00000000..c792a8b4 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml @@ -0,0 +1,6 @@ +--- +kind: AdmissionConfiguration +apiVersion: apiserver.k8s.io/v1alpha1 +plugins: +- name: EventRateLimit + path: eventconfig.yaml \ No newline at end of file diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml new file mode 100644 index 00000000..ae789689 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml @@ -0,0 +1,7 @@ +--- +kind: Configuration +apiVersion: eventratelimit.admission.k8s.io/v1alpha1 +limits: +- type: Server + qps: 100 + burst: 1000 \ No newline at end of file diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml index adfa41ef..3e6e48b1 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml @@ -122,8 +122,6 @@ spec: - "{{ argument }}" {%- endfor %} - --advertise-address={{ config['Genesis:ip'] }} - - --authorization-mode=Node,RBAC - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds - --anonymous-auth=false - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem @@ -132,15 +130,14 @@ spec: - --insecure-port=8080 - --secure-port=6444 - --bind-address=0.0.0.0 - - --runtime-config=batch/v2alpha1=true - --allow-privileged=true - --etcd-servers=https://localhost:12379 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem - - --service-cluster-ip-range={{ config['KubernetesNetwork:kubernetes.service_cidr'] }} - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub + - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem env: diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml index ef27e8b5..606f0f3e 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml @@ -20,8 +20,6 @@ spec: - "{{ argument }}" {%- endfor %} - --advertise-address={{ config['Genesis:ip'] }} - - --authorization-mode=Node,RBAC - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds - --anonymous-auth=false - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem @@ -30,15 +28,14 @@ spec: - --insecure-port=0 - --bind-address=0.0.0.0 - --secure-port=6443 - - --runtime-config=batch/v2alpha1=true - --allow-privileged=true - --etcd-servers=https://localhost:2379 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem - - --service-cluster-ip-range={{ config['KubernetesNetwork:kubernetes.service_cidr'] }} - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub + - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem volumeMounts: