From 04da7585ffe846ccadb96110ed7a27531ca390ce Mon Sep 17 00:00:00 2001
From: Mark Burnett <mark.m.burnett@gmail.com>
Date: Tue, 4 Dec 2018 07:47:29 -0600
Subject: [PATCH] Refactor API server

This change accomplishes 2 primary things:
1. It generalizes work to enable the EventRateLimit admission plugin.
2. It restructures the anchor so that during an upgrade an "old" anchor
   does not try to coordinate the injection of "new" data from
   configmaps/secrets.

It also includes these ancillary changes:
* Clean up apiserver argument specification in the chart.
* De-duplicate and realign apiserver arguments in bootstrapping templates.

It has the side effects of:
* Adding a new field, ".apiserver.arguments" to the Genesis config,
  which will be the preferred way to configure bootstrapping apiservers
  going forward (in lieu of command_prefix).

Change-Id: I33cfe80ee8e29cd79e479a7985e3c098a2288fda
---
 charts/apiserver/templates/bin/_anchor.tpl    |  54 +++-
 charts/apiserver/templates/configmap-etc.yaml |  29 +-
 .../etc/_kubernetes-apiserver.yaml.tpl        |  35 +--
 charts/apiserver/values.yaml                  | 149 ++++++----
 examples/basic/Genesis.yaml                   |  25 +-
 examples/basic/armada-resources.yaml          |   9 -
 promenade/config.py                           |   2 +-
 promenade/schemas/Genesis.yaml                |   4 +
 .../templates/include/genesis-apiserver.yaml  |  18 ++
 .../etc/genesis/apiserver/acconfig.yaml       |   6 -
 .../etc/genesis/apiserver/eventconfig.yaml    |   7 -
 .../manifests/bootstrap-armada.yaml           | 256 ++++++++----------
 .../manifests/kubernetes-apiserver.yaml       |  21 +-
 13 files changed, 321 insertions(+), 294 deletions(-)
 create mode 100644 promenade/templates/include/genesis-apiserver.yaml
 delete mode 100644 promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml
 delete mode 100644 promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml

diff --git a/charts/apiserver/templates/bin/_anchor.tpl b/charts/apiserver/templates/bin/_anchor.tpl
index c311ffa0..904a4670 100644
--- a/charts/apiserver/templates/bin/_anchor.tpl
+++ b/charts/apiserver/templates/bin/_anchor.tpl
@@ -15,26 +15,54 @@
 
 set -x
 
-compare_copy_files() {
+snapshot_files() {
+    SNAPSHOT_DIR=${1}
+    {{ range $dest, $source := .Values.const.files_to_copy }}
+    mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
+    cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
+    {{- end }}
+    {{ range $key, $val := .Values.conf }}
+    cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
+    {{- end }}
+}
 
-    {{range .Values.anchor.files_to_copy}}
-    if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
-        mkdir -p $(dirname /host{{ .dest }})
-        cp {{ .source }} /host{{ .dest }}
-        chmod go-rwx /host{{ .dest }}
+compare_copy_files() {
+    SNAPSHOT_DIR=${1}
+    {{ range $dest, $source := .Values.const.files_to_copy }}
+    SRC="${SNAPSHOT_DIR}{{ $dest }}"
+    DEST="/host{{ $dest }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
     fi
-    {{end}}
+    {{- end}}
+    {{ range $key, $val := .Values.conf }}
+    SRC="${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
+    DEST="/host/etc/kubernetes/apiserver/{{ $val.file }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
+    fi
+    {{- end }}
 }
 
 cleanup() {
-
-    {{range .Values.anchor.files_to_copy}}
-    rm -f /host{{ .dest }}
-    {{end}}
+    {{- range $dest, $source := .Values.const.files_to_copy }}
+    rm -f "/host{{ $dest }}"
+    {{- end }}
+    {{  range $key, $val := .Values.conf }}
+    rm -f "/host/{{ $val.file }}"
+    {{- end }}
 }
 
-while true; do
 
+SNAPSHOT_DIR=$(mktemp -d)
+
+snapshot_files "${SNAPSHOT_DIR}"
+
+while true; do
     if [ -e /tmp/stop ]; then
         echo Stopping
         cleanup
@@ -43,7 +71,7 @@ while true; do
 
     # Compare and replace files on Genesis host if needed
     # Copy files to other master nodes
-    compare_copy_files
+    compare_copy_files "${SNAPSHOT_DIR}"
 
     sleep {{ .Values.anchor.period }}
 done
diff --git a/charts/apiserver/templates/configmap-etc.yaml b/charts/apiserver/templates/configmap-etc.yaml
index 75a22eac..016290fd 100644
--- a/charts/apiserver/templates/configmap-etc.yaml
+++ b/charts/apiserver/templates/configmap-etc.yaml
@@ -17,34 +17,19 @@ limitations under the License.
 {{- if .Values.manifests.configmap_etc }}
 {{- $envAll := . }}
 
-{{/* This slightly involved merge of AC config files into the anchor
-     files uses HTK merge, as straighforward appends result in duplicates. */}}
-{{- $_ := set .Values "_ac_files_to_copy" list }}
-{{- range $key, $val := .Values.conf.admission_controllers }}
-  {{- $source := printf "/tmp/etc/%s" $key }}
-  {{- $dest := printf "/etc/kubernetes/apiserver/%s" $key }}
-  {{- $file_to_copy := dict "source" $source "dest" $dest }}
-  {{- $ac_files_to_copy := append $.Values._ac_files_to_copy $file_to_copy }}
-  {{- $_ := set $.Values "_ac_files_to_copy" $ac_files_to_copy }}
-{{- end }}
-{{ $all_files_to_copy := dict }}
-{{ $_ := set $all_files_to_copy "values" (tuple .Values.anchor.files_to_copy .Values._ac_files_to_copy) }}
-{{ $_ := $all_files_to_copy | include "helm-toolkit.utils.merge" }}
-{{ $_ := set .Values.anchor "files_to_copy" $all_files_to_copy.result }}
-
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ .Values.service.name }}-etc
 data:
-  kubernetes-apiserver.yaml: |+
+  kubernetes-apiserver.yaml: |
 {{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  kubeconfig.yaml: |+
+  kubeconfig.yaml: |
 {{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{/* Dynamically add config files for admission controllers */}}
-{{ range $key, $val := .Values.conf.admission_controllers }}
-  {{ $key }}: |+
-{{ toYaml $val | indent 4 }}
-{{ end }}
+{{/* Dynamically added config files */}}
+{{- range $key, $val := .Values.conf }}
+  {{ $val.file }}: |
+{{ toYaml $val.content | indent 4 }}
+{{- end }}
 {{- end }}
diff --git a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
index daf04e13..73f6ccfc 100644
--- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
+++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
@@ -42,30 +42,25 @@ spec:
               fieldPath: spec.nodeName
         - name: KUBECONFIG
           value: /etc/kubernetes/apiserver/kubeconfig.yaml
+        - name: APISERVER_PORT
+          value: {{ .Values.network.kubernetes_apiserver.port | quote }}
+        - name: ETCD_ENDPOINTS
+          value: {{ .Values.apiserver.etcd.endpoints | quote }}
 
       command:
-        {{- range .Values.command_prefix }}
+        {{- range .Values.const.command_prefix }}
         - {{ . }}
         {{- end }}
-        - --advertise-address=$(POD_IP)
-        - --anonymous-auth=false
-        - --bind-address=0.0.0.0
-        - --secure-port={{ .Values.network.kubernetes_apiserver.port }}
-        - --insecure-port=0
-        - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
-        - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
-        - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
-        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-        - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
-        - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
-        - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
-        - --etcd-servers={{ .Values.apiserver.etcd.endpoints }}
-        - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
-        - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
-        - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
-        - --allow-privileged=true
-        - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
-        - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
+        {{- range .Values.apiserver.arguments }}
+        - {{ . }}
+        {{- end }}
+        {{- range $key, $val := .Values.conf }}
+        {{- if hasKey $val "command_options" }}
+        {{- range $val.command_options }}
+        - {{ . }}
+        {{- end }}
+        {{- end }}
+        {{- end }}
 
       ports:
         - containerPort: {{ .Values.network.kubernetes_apiserver.port }}
diff --git a/charts/apiserver/values.yaml b/charts/apiserver/values.yaml
index 9c0556bd..b7c5ecf7 100644
--- a/charts/apiserver/values.yaml
+++ b/charts/apiserver/values.yaml
@@ -14,6 +14,45 @@
 
 release_group: null
 
+# NOTE(mark-burnett): These values are not really configurable -- they live
+# here to keep the templates cleaner.
+const:
+  command_prefix:
+    - /apiserver
+    - --advertise-address=$(POD_IP)
+    - --allow-privileged=true
+    - --anonymous-auth=false
+    - --bind-address=0.0.0.0
+    - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+    - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
+    - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
+    - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
+    - --etcd-servers=$(ETCD_ENDPOINTS)
+    - --insecure-port=0
+    - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+    - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
+    - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
+    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+    - --secure-port=$(APISERVER_PORT)
+    - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
+    - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
+    - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
+
+  files_to_copy:
+    # NOTE(mark-burnett): These are (host dest): (container source) pairs
+    /etc/kubernetes/apiserver/kubeconfig.yaml: /tmp/etc/kubeconfig.yaml
+    /etc/kubernetes/apiserver/pki/apiserver-key.pem: /keys/apiserver-key.pem
+    /etc/kubernetes/apiserver/pki/apiserver.pem: /certs/apiserver.pem
+    /etc/kubernetes/apiserver/pki/cluster-ca.pem: /certs/cluster-ca.pem
+    /etc/kubernetes/apiserver/pki/etcd-client-ca.pem: /certs/etcd-client-ca.pem
+    /etc/kubernetes/apiserver/pki/etcd-client-key.pem: /keys/etcd-client-key.pem
+    /etc/kubernetes/apiserver/pki/etcd-client.pem: /certs/etcd-client.pem
+    /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem: /certs/kubelet-client-ca.pem
+    /etc/kubernetes/apiserver/pki/kubelet-client-key.pem: /keys/kubelet-client-key.pem
+    /etc/kubernetes/apiserver/pki/kubelet-client.pem: /certs/kubelet-client.pem
+    /etc/kubernetes/apiserver/pki/service-account.pub: /certs/service-account.pub
+    /etc/kubernetes/manifests/kubernetes-apiserver.yaml: /tmp/etc/kubernetes-apiserver.yaml
+
 images:
   tags:
     anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.11
@@ -30,65 +69,58 @@ anchor:
   kubelet:
     manifest_path: /etc/kubernetes/manifests
   period: 15
-  files_to_copy:
-    - source: /certs/apiserver.pem
-      dest: /etc/kubernetes/apiserver/pki/apiserver.pem
-    - source: /certs/kubelet-client.pem
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client.pem
-    - source: /certs/kubelet-client-ca.pem
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
-    - source: /certs/cluster-ca.pem
-      dest: /etc/kubernetes/apiserver/pki/cluster-ca.pem
-    - source: /certs/etcd-client-ca.pem
-      dest: /etc/kubernetes/apiserver/pki/etcd-client-ca.pem
-    - source: /certs/etcd-client.pem
-      dest: /etc/kubernetes/apiserver/pki/etcd-client.pem
-    - source: /certs/service-account.pub
-      dest: /etc/kubernetes/apiserver/pki/service-account.pub
-    - source: /keys/apiserver-key.pem
-      dest: /etc/kubernetes/apiserver/pki/apiserver-key.pem
-    - source: /keys/kubelet-client-key.pem
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client-key.pem
-    - source: /keys/etcd-client-key.pem
-      dest: /etc/kubernetes/apiserver/pki/etcd-client-key.pem
-    - source: /tmp/etc/kubernetes-apiserver.yaml
-      dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml
-    - source: /tmp/etc/kubeconfig.yaml
-      dest: /etc/kubernetes/apiserver/kubeconfig.yaml
-    # Note: config files for admission controllers are added to this dynamically
-
-command_prefix:
-  - /apiserver
-  - --authorization-mode=Node,RBAC
-  - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
-  - --service-cluster-ip-range=10.96.0.0/16
-  - --endpoint-reconciler-type=lease
-  # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
-  - --repair-malformed-updates=false
-
-apiserver:
-  host_etc_path: /etc/kubernetes/apiserver
-  etcd:
-    endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local
 
 conf:
-  # Admission controllers config files are generated dynamically based on the
-  # config below, as they are specific to particular ACs that may be
-  # configured by the operator (or added by k8s in the future).
-  admission_controllers:
-    eventconfig.yaml:
-      kind: Configuration
-      apiVersion: eventratelimit.admission.k8s.io/v1alpha1
-      limits:
-      - type: Server
-        qps: 100
-        burst: 1000
-    acconfig.yaml:
-      kind: AdmissionConfiguration
-      apiVersion: apiserver.k8s.io/v1alpha1
-      plugins:
-      - name: EventRateLimit
-        path: eventconfig.yaml
+# Uncomment any of the below to enable the file placement and associated apiserver
+# command line options
+#
+#  acconfig:
+#    file: acconfig.yaml
+#    command_options:
+#      - '--admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml'
+#      - '--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit'
+#    content:
+#      kind: AdmissionConfiguration
+#      apiVersion: apiserver.k8s.io/v1alpha1
+#      plugins:
+#        - name: EventRateLimit
+#          path: eventconfig.yaml
+#  eventconfig:
+#    file: eventconfig.yaml
+#    command_options:
+#      - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml'
+#    content:
+#      kind: Configuration
+#      apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+#      limits:
+#        - type: Server
+#          qps: 1000
+#          burst: 10000
+#  encryption_provider:
+#    file: encryption_provider.yaml
+#    command_option: ''
+#    content:
+#      kind: EncryptionConfig
+#      apiVersion: v1
+#      resources:
+#        - resources:
+#            - 'secrets'
+#          providers:
+#            - identity: {}
+
+apiserver:
+  arguments:
+    - --authorization-mode=Node,RBAC
+    - --service-cluster-ip-range=10.96.0.0/16
+    - --endpoint-reconciler-type=lease
+    - --feature-gates=PodShareProcessNamespace=true
+    # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
+    - --repair-malformed-updates=false
+    - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction
+    - --v=3
+  etcd:
+    endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local
+  host_etc_path: /etc/kubernetes/apiserver
 
 network:
   kubernetes_apiserver:
@@ -130,7 +162,6 @@ secrets:
       cert: null
       key: null
 
-
 # typically overriden by environmental
 # values, but should include all endpoints
 # required by this chart
@@ -170,7 +201,7 @@ pod:
     upgrades:
       daemonsets:
         pod_replacement_strategy: RollingUpdate
-        kubernetes_apiserver:
+        kubernetes-apiserver-anchor:
           enabled: false
           min_ready_seconds: 0
           max_unavailable: 1
diff --git a/examples/basic/Genesis.yaml b/examples/basic/Genesis.yaml
index 823ae70a..ddc19168 100644
--- a/examples/basic/Genesis.yaml
+++ b/examples/basic/Genesis.yaml
@@ -11,15 +11,16 @@ data:
   hostname: n0
   ip: 192.168.77.10
   apiserver:
-    command_prefix:
-      - /apiserver
+    arguments:
       - --authorization-mode=Node,RBAC
-      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
       - --service-cluster-ip-range=10.96.0.0/16
       - --endpoint-reconciler-type=lease
       - --feature-gates=PodShareProcessNamespace=true
       # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
       - --repair-malformed-updates=false
+      - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
+      - --v=3
   armada:
     target_manifest: cluster-bootstrap
   labels:
@@ -45,4 +46,22 @@ data:
     - path: /var/lib/anchor/calico-etcd-bootstrap
       content: "# placeholder for triggering calico etcd bootstrapping"
       mode: 0644
+    # NOTE(mark-burnett): These are referenced by the apiserver arguments above.
+    - path: /etc/genesis/apiserver/acconfig.yaml
+      mode: 0444
+      content: |
+        kind: AdmissionConfiguration
+        apiVersion: apiserver.k8s.io/v1alpha1
+        plugins:
+          - name: EventRateLimit
+            path: eventconfig.yaml
+    - path: /etc/genesis/apiserver/eventconfig.yaml
+      mode: 0444
+      content: |
+        kind: Configuration
+        apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+        limits:
+          - type: Server
+            qps: 1000
+            burst: 10000
 ...
diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml
index f99fdd30..8df50a16 100644
--- a/examples/basic/armada-resources.yaml
+++ b/examples/basic/armada-resources.yaml
@@ -719,15 +719,6 @@ data:
   upgrade:
     no_hooks: true
   values:
-    command_prefix:
-      - /apiserver
-      - --authorization-mode=Node,RBAC
-      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
-      - --service-cluster-ip-range=10.96.0.0/16
-      - --endpoint-reconciler-type=lease
-      - --feature-gates=PodShareProcessNamespace=true
-      # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
-      - --repair-malformed-updates=false
     apiserver:
       etcd:
         endpoints: https://127.0.0.1:2378
diff --git a/promenade/config.py b/promenade/config.py
index 79febba1..6553077d 100644
--- a/promenade/config.py
+++ b/promenade/config.py
@@ -241,7 +241,7 @@ class Configuration:
 
     def bootstrap_apiserver_prefix(self):
         return self.get_path('Genesis:apiserver.command_prefix',
-                             ['/apiserver', '--apiserver-count=2', '--v=5'])
+                             ['/apiserver'])
 
 
 def _matches_filter(document, *, schema, labels, name):
diff --git a/promenade/schemas/Genesis.yaml b/promenade/schemas/Genesis.yaml
index 12f9b5b5..021f3c38 100644
--- a/promenade/schemas/Genesis.yaml
+++ b/promenade/schemas/Genesis.yaml
@@ -71,6 +71,10 @@ data:
           type: array
           items:
             type: string
+        arguments:
+          type: array
+          items:
+            type: string
       additionalProperties: false
 
     files:
diff --git a/promenade/templates/include/genesis-apiserver.yaml b/promenade/templates/include/genesis-apiserver.yaml
new file mode 100644
index 00000000..4314a61c
--- /dev/null
+++ b/promenade/templates/include/genesis-apiserver.yaml
@@ -0,0 +1,18 @@
+        - --advertise-address={{ config['Genesis:ip'] }}
+        - --allow-privileged=true
+        - --anonymous-auth=false
+        - --bind-address=0.0.0.0
+        - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+        - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
+        - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
+        - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
+        - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
+        - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
+        - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
+        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+        - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
+        - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
+        - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
+        {%- for argument in config.get_path('Genesis:apiserver.arguments', []) %}
+        - "{{ argument }}"
+        {%- endfor %}
diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml
deleted file mode 100644
index c792a8b4..00000000
--- a/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-kind: AdmissionConfiguration
-apiVersion: apiserver.k8s.io/v1alpha1
-plugins:
-- name: EventRateLimit
-  path: eventconfig.yaml
\ No newline at end of file
diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml
deleted file mode 100644
index ae789689..00000000
--- a/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-kind: Configuration
-apiVersion: eventratelimit.admission.k8s.io/v1alpha1
-limits:
-- type: Server
-  qps: 100
-  burst: 1000
\ No newline at end of file
diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
index d122d570..e9051aa4 100644
--- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
+++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
@@ -11,146 +11,130 @@ spec:
   dnsPolicy: Default
   hostNetwork: true
   containers:
-  - env:
-    - name: TILLER_NAMESPACE
-      value: kube-system
-    image: {{ config['Genesis:images.helm.tiller'] }}
-    command:
-      - /tiller
-      - -logtostderr
-      - -v
-      - "99"
-    imagePullPolicy: IfNotPresent
-    livenessProbe:
-      failureThreshold: 3
-      httpGet:
-        path: /liveness
-        port: 44135
-        scheme: HTTP
-      initialDelaySeconds: 1
-      periodSeconds: 10
-      successThreshold: 1
-      timeoutSeconds: 1
-    name: tiller
-    ports:
-    - containerPort: 44134
+    - env:
+      - name: TILLER_NAMESPACE
+        value: kube-system
+      image: {{ config['Genesis:images.helm.tiller'] }}
+      command:
+        - /tiller
+        - -logtostderr
+        - -v
+        - "99"
+      imagePullPolicy: IfNotPresent
+      livenessProbe:
+        failureThreshold: 3
+        httpGet:
+          path: /liveness
+          port: 44135
+          scheme: HTTP
+        initialDelaySeconds: 1
+        periodSeconds: 10
+        successThreshold: 1
+        timeoutSeconds: 1
       name: tiller
-      protocol: TCP
-    readinessProbe:
-      failureThreshold: 3
-      httpGet:
-        path: /readiness
-        port: 44135
-        scheme: HTTP
-      initialDelaySeconds: 1
-      periodSeconds: 10
-      successThreshold: 1
-      timeoutSeconds: 1
-    resources: {}
-    terminationMessagePath: /dev/termination-log
-    terminationMessagePolicy: File
-  - name: armada
-    image: {{ config['Genesis:images.armada'] }}
-    securityContext:
-      runAsUser: 0
-    command:
-      - /bin/bash
-      - -c
-      - |-
-        set -x
+      ports:
+      - containerPort: 44134
+        name: tiller
+        protocol: TCP
+      readinessProbe:
+        failureThreshold: 3
+        httpGet:
+          path: /readiness
+          port: 44135
+          scheme: HTTP
+        initialDelaySeconds: 1
+        periodSeconds: 10
+        successThreshold: 1
+        timeoutSeconds: 1
+      resources: {}
+      terminationMessagePath: /dev/termination-log
+      terminationMessagePolicy: File
+    - name: armada
+      image: {{ config['Genesis:images.armada'] }}
+      securityContext:
+        runAsUser: 0
+      command:
+        - /bin/bash
+        - -c
+        - |-
+          set -x
 
-        while true; do
-            sleep 10
-            if armada \
-                    apply \
-                    --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
-                    --tiller-host 127.0.0.1 \
-                    /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
-                break
-            fi
-        done
+          while true; do
+              sleep 10
+              if armada \
+                      apply \
+                      --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
+                      --tiller-host 127.0.0.1 \
+                      /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
+                  break
+              fi
+          done
+          touch /ipc/armada-done
+          sleep 10000
+      env:
+        - name: ARMADA_LOGFILE
+          value: /tmp/log/bootstrap-armada.log
+  {%- if config['KubernetesNetwork:proxy.url'] is defined %}
+        - name: HTTP_PROXY
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
+        - name: HTTPS_PROXY
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
+        - name: NO_PROXY
+          value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
+        - name: http_proxy
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
+        - name: https_proxy
+          value: {{ config['KubernetesNetwork:proxy.url'] }}
+        - name: no_proxy
+          value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
+  {%- endif %}
+      volumeMounts:
+        - name: assets
+          mountPath: /etc/genesis/armada/assets
+        - name: auth
+          mountPath: /root/.kube
+        - name: ipc
+          mountPath: /ipc
+        - name: log
+          mountPath: /tmp/log
+    - name: monitor
+      image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
+      command:
+        - /bin/sh
+        - -c
+        - |-
+          set -x
 
-        touch /ipc/armada-done
-        sleep 10000
-    env:
-      - name: ARMADA_LOGFILE
-        value: /tmp/log/bootstrap-armada.log
-{%- if config['KubernetesNetwork:proxy.url'] is defined %}
-      - name: HTTP_PROXY
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
-      - name: HTTPS_PROXY
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
-      - name: NO_PROXY
-        value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
-      - name: http_proxy
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
-      - name: https_proxy
-        value: {{ config['KubernetesNetwork:proxy.url'] }}
-      - name: no_proxy
-        value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
-{%- endif %}
-    volumeMounts:
-      - name: assets
-        mountPath: /etc/genesis/armada/assets
-      - name: auth
-        mountPath: /root/.kube
-      - name: ipc
-        mountPath: /ipc
-      - name: log
-        mountPath: /tmp/log
-  - name: monitor
-    image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
-    command:
-      - /bin/sh
-      - -c
-      - |-
-        set -x
+          while ! [ -e /ipc/armada-done ]; do
+            sleep 5
+          done
 
-        while ! [ -e /ipc/armada-done ]; do
-          sleep 5
-        done
-
-        rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml
-        sleep 10000
-    volumeMounts:
-      - name: ipc
-        mountPath: /ipc
-      - name: manifest
-        mountPath: /etc/kubernetes/manifests
-  - name: kubectl-apiserver
-    image: {{ config['Genesis:images.kubernetes.apiserver'] }}
-    command:
-      {%- for argument in config.bootstrap_apiserver_prefix() %}
-      - "{{ argument }}"
-      {%- endfor %}
-      - --advertise-address={{ config['Genesis:ip'] }}
-      - --anonymous-auth=false
-      - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
-      - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
-      - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem
-      - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem
-      - --insecure-port=8080
-      - --secure-port=6444
-      - --bind-address=0.0.0.0
-      - --allow-privileged=true
-      - --etcd-servers=https://localhost:12379
-      - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
-      - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
-      - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
-      - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-      - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
-      - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
-      - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
-      - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
-    env:
-      - name: KUBECONFIG
-        value: /etc/kubernetes/admin/config
-    volumeMounts:
-      - name: auth
-        mountPath: /etc/kubernetes/admin
-      - name: config
-        mountPath: /etc/kubernetes/apiserver
-        readOnly: true
+          rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml
+          sleep 10000
+      volumeMounts:
+        - name: ipc
+          mountPath: /ipc
+        - name: manifest
+          mountPath: /etc/kubernetes/manifests
+    - name: kubectl-apiserver
+      image: {{ config['Genesis:images.kubernetes.apiserver'] }}
+      command:
+        {%- for argument in config.bootstrap_apiserver_prefix() %}
+        - "{{ argument }}"
+        {%- endfor %}
+{% include "genesis-apiserver.yaml" with context %}
+        - --etcd-servers=https://localhost:12379
+        - --insecure-port=8080
+        - --secure-port=6444
+      env:
+        - name: KUBECONFIG
+          value: /etc/kubernetes/admin/config
+      volumeMounts:
+        - name: auth
+          mountPath: /etc/kubernetes/admin
+        - name: config
+          mountPath: /etc/kubernetes/apiserver
+          readOnly: true
   volumes:
     - name: assets
       hostPath:
diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml
index 606f0f3e..4113327b 100644
--- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml
+++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml
@@ -19,25 +19,10 @@ spec:
         {%- for argument in config.bootstrap_apiserver_prefix() %}
         - "{{ argument }}"
         {%- endfor %}
-        - --advertise-address={{ config['Genesis:ip'] }}
-        - --anonymous-auth=false
-        - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
-        - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
-        - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
-        - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
-        - --insecure-port=0
-        - --bind-address=0.0.0.0
-        - --secure-port=6443
-        - --allow-privileged=true
+{% include "genesis-apiserver.yaml" with context %}
         - --etcd-servers=https://localhost:2379
-        - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
-        - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
-        - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
-        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-        - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
-        - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
-        - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
-        - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
+        - --insecure-port=0
+        - --secure-port=6443
       volumeMounts:
         - name: config
           mountPath: /etc/kubernetes/apiserver