summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMark Burnett <mark.m.burnett@gmail.com>2018-12-04 07:47:29 -0600
committerScott Hussey <sh8121@att.com>2019-01-10 16:31:50 -0600
commit04da7585ffe846ccadb96110ed7a27531ca390ce (patch)
tree8b83cc7d8d2c3a93ef423c00eb7aaaf848ca15b6
parentb5a05dc762f1b8ca05de90febbef519ea336495b (diff)
Refactor API serverHEADmaster
This change accomplishes 2 primary things: 1. It generalizes work to enable the EventRateLimit admission plugin. 2. It restructures the anchor so that during an upgrade an "old" anchor does not try to coordinate the injection of "new" data from configmaps/secrets. It also includes these ancillary changes: * Clean up apiserver argument specification in the chart. * De-duplicate and realign apiserver arguments in bootstrapping templates. It has the side effects of: * Adding a new field, ".apiserver.arguments" to the Genesis config, which will be the preferred way to configure bootstrapping apiservers going forward (in lieu of command_prefix). Change-Id: I33cfe80ee8e29cd79e479a7985e3c098a2288fda
Notes
Notes (review): Code-Review+2: Matt McEuen <matt.mceuen@att.com> Code-Review+2: Bryan Strassner <strassner.bryan@gmail.com> Workflow+1: Matt McEuen <matt.mceuen@att.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Mon, 14 Jan 2019 19:19:42 +0000 Reviewed-on: https://review.openstack.org/622586 Project: openstack/airship-promenade Branch: refs/heads/master
-rw-r--r--charts/apiserver/templates/bin/_anchor.tpl54
-rw-r--r--charts/apiserver/templates/configmap-etc.yaml29
-rw-r--r--charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl35
-rw-r--r--charts/apiserver/values.yaml143
-rw-r--r--examples/basic/Genesis.yaml25
-rw-r--r--examples/basic/armada-resources.yaml9
-rw-r--r--promenade/config.py2
-rw-r--r--promenade/schemas/Genesis.yaml4
-rw-r--r--promenade/templates/include/genesis-apiserver.yaml18
-rw-r--r--promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml6
-rw-r--r--promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml7
-rw-r--r--promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml256
-rw-r--r--promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml19
13 files changed, 317 insertions, 290 deletions
diff --git a/charts/apiserver/templates/bin/_anchor.tpl b/charts/apiserver/templates/bin/_anchor.tpl
index c311ffa..904a467 100644
--- a/charts/apiserver/templates/bin/_anchor.tpl
+++ b/charts/apiserver/templates/bin/_anchor.tpl
@@ -15,26 +15,54 @@
15 15
16set -x 16set -x
17 17
18compare_copy_files() { 18snapshot_files() {
19 SNAPSHOT_DIR=${1}
20 {{ range $dest, $source := .Values.const.files_to_copy }}
21 mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
22 cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
23 {{- end }}
24 {{ range $key, $val := .Values.conf }}
25 cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
26 {{- end }}
27}
19 28
20 {{range .Values.anchor.files_to_copy}} 29compare_copy_files() {
21 if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then 30 SNAPSHOT_DIR=${1}
22 mkdir -p $(dirname /host{{ .dest }}) 31 {{ range $dest, $source := .Values.const.files_to_copy }}
23 cp {{ .source }} /host{{ .dest }} 32 SRC="${SNAPSHOT_DIR}{{ $dest }}"
24 chmod go-rwx /host{{ .dest }} 33 DEST="/host{{ $dest }}"
34 if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
35 mkdir -p $(dirname "${DEST}")
36 cp "${SRC}" "${DEST}"
37 chmod go-rwx "${DEST}"
25 fi 38 fi
26 {{end}} 39 {{- end}}
40 {{ range $key, $val := .Values.conf }}
41 SRC="${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
42 DEST="/host/etc/kubernetes/apiserver/{{ $val.file }}"
43 if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
44 mkdir -p $(dirname "${DEST}")
45 cp "${SRC}" "${DEST}"
46 chmod go-rwx "${DEST}"
47 fi
48 {{- end }}
27} 49}
28 50
29cleanup() { 51cleanup() {
30 52 {{- range $dest, $source := .Values.const.files_to_copy }}
31 {{range .Values.anchor.files_to_copy}} 53 rm -f "/host{{ $dest }}"
32 rm -f /host{{ .dest }} 54 {{- end }}
33 {{end}} 55 {{ range $key, $val := .Values.conf }}
56 rm -f "/host/{{ $val.file }}"
57 {{- end }}
34} 58}
35 59
36while true; do
37 60
61SNAPSHOT_DIR=$(mktemp -d)
62
63snapshot_files "${SNAPSHOT_DIR}"
64
65while true; do
38 if [ -e /tmp/stop ]; then 66 if [ -e /tmp/stop ]; then
39 echo Stopping 67 echo Stopping
40 cleanup 68 cleanup
@@ -43,7 +71,7 @@ while true; do
43 71
44 # Compare and replace files on Genesis host if needed 72 # Compare and replace files on Genesis host if needed
45 # Copy files to other master nodes 73 # Copy files to other master nodes
46 compare_copy_files 74 compare_copy_files "${SNAPSHOT_DIR}"
47 75
48 sleep {{ .Values.anchor.period }} 76 sleep {{ .Values.anchor.period }}
49done 77done
diff --git a/charts/apiserver/templates/configmap-etc.yaml b/charts/apiserver/templates/configmap-etc.yaml
index 75a22ea..016290f 100644
--- a/charts/apiserver/templates/configmap-etc.yaml
+++ b/charts/apiserver/templates/configmap-etc.yaml
@@ -17,34 +17,19 @@ limitations under the License.
17{{- if .Values.manifests.configmap_etc }} 17{{- if .Values.manifests.configmap_etc }}
18{{- $envAll := . }} 18{{- $envAll := . }}
19 19
20{{/* This slightly involved merge of AC config files into the anchor
21 files uses HTK merge, as straighforward appends result in duplicates. */}}
22{{- $_ := set .Values "_ac_files_to_copy" list }}
23{{- range $key, $val := .Values.conf.admission_controllers }}
24 {{- $source := printf "/tmp/etc/%s" $key }}
25 {{- $dest := printf "/etc/kubernetes/apiserver/%s" $key }}
26 {{- $file_to_copy := dict "source" $source "dest" $dest }}
27 {{- $ac_files_to_copy := append $.Values._ac_files_to_copy $file_to_copy }}
28 {{- $_ := set $.Values "_ac_files_to_copy" $ac_files_to_copy }}
29{{- end }}
30{{ $all_files_to_copy := dict }}
31{{ $_ := set $all_files_to_copy "values" (tuple .Values.anchor.files_to_copy .Values._ac_files_to_copy) }}
32{{ $_ := $all_files_to_copy | include "helm-toolkit.utils.merge" }}
33{{ $_ := set .Values.anchor "files_to_copy" $all_files_to_copy.result }}
34
35--- 20---
36apiVersion: v1 21apiVersion: v1
37kind: ConfigMap 22kind: ConfigMap
38metadata: 23metadata:
39 name: {{ .Values.service.name }}-etc 24 name: {{ .Values.service.name }}-etc
40data: 25data:
41 kubernetes-apiserver.yaml: |+ 26 kubernetes-apiserver.yaml: |
42{{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} 27{{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
43 kubeconfig.yaml: |+ 28 kubeconfig.yaml: |
44{{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} 29{{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
45{{/* Dynamically add config files for admission controllers */}} 30{{/* Dynamically added config files */}}
46{{ range $key, $val := .Values.conf.admission_controllers }} 31{{- range $key, $val := .Values.conf }}
47 {{ $key }}: |+ 32 {{ $val.file }}: |
48{{ toYaml $val | indent 4 }} 33{{ toYaml $val.content | indent 4 }}
49{{ end }} 34{{- end }}
50{{- end }} 35{{- end }}
diff --git a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
index daf04e1..73f6ccf 100644
--- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
+++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
@@ -42,30 +42,25 @@ spec:
42 fieldPath: spec.nodeName 42 fieldPath: spec.nodeName
43 - name: KUBECONFIG 43 - name: KUBECONFIG
44 value: /etc/kubernetes/apiserver/kubeconfig.yaml 44 value: /etc/kubernetes/apiserver/kubeconfig.yaml
45 - name: APISERVER_PORT
46 value: {{ .Values.network.kubernetes_apiserver.port | quote }}
47 - name: ETCD_ENDPOINTS
48 value: {{ .Values.apiserver.etcd.endpoints | quote }}
45 49
46 command: 50 command:
47 {{- range .Values.command_prefix }} 51 {{- range .Values.const.command_prefix }}
48 - {{ . }} 52 - {{ . }}
49 {{- end }} 53 {{- end }}
50 - --advertise-address=$(POD_IP) 54 {{- range .Values.apiserver.arguments }}
51 - --anonymous-auth=false 55 - {{ . }}
52 - --bind-address=0.0.0.0 56 {{- end }}
53 - --secure-port={{ .Values.network.kubernetes_apiserver.port }} 57 {{- range $key, $val := .Values.conf }}
54 - --insecure-port=0 58 {{- if hasKey $val "command_options" }}
55 - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem 59 {{- range $val.command_options }}
56 - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem 60 - {{ . }}
57 - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem 61 {{- end }}
58 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname 62 {{- end }}
59 - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem 63 {{- end }}
60 - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
61 - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
62 - --etcd-servers={{ .Values.apiserver.etcd.endpoints }}
63 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
64 - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
65 - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
66 - --allow-privileged=true
67 - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
68 - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
69 64
70 ports: 65 ports:
71 - containerPort: {{ .Values.network.kubernetes_apiserver.port }} 66 - containerPort: {{ .Values.network.kubernetes_apiserver.port }}
diff --git a/charts/apiserver/values.yaml b/charts/apiserver/values.yaml
index 9c0556b..b7c5ecf 100644
--- a/charts/apiserver/values.yaml
+++ b/charts/apiserver/values.yaml
@@ -14,6 +14,45 @@
14 14
15release_group: null 15release_group: null
16 16
17# NOTE(mark-burnett): These values are not really configurable -- they live
18# here to keep the templates cleaner.
19const:
20 command_prefix:
21 - /apiserver
22 - --advertise-address=$(POD_IP)
23 - --allow-privileged=true
24 - --anonymous-auth=false
25 - --bind-address=0.0.0.0
26 - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
27 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
28 - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
29 - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
30 - --etcd-servers=$(ETCD_ENDPOINTS)
31 - --insecure-port=0
32 - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
33 - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
34 - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
35 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
36 - --secure-port=$(APISERVER_PORT)
37 - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
38 - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
39 - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
40
41 files_to_copy:
42 # NOTE(mark-burnett): These are (host dest): (container source) pairs
43 /etc/kubernetes/apiserver/kubeconfig.yaml: /tmp/etc/kubeconfig.yaml
44 /etc/kubernetes/apiserver/pki/apiserver-key.pem: /keys/apiserver-key.pem
45 /etc/kubernetes/apiserver/pki/apiserver.pem: /certs/apiserver.pem
46 /etc/kubernetes/apiserver/pki/cluster-ca.pem: /certs/cluster-ca.pem
47 /etc/kubernetes/apiserver/pki/etcd-client-ca.pem: /certs/etcd-client-ca.pem
48 /etc/kubernetes/apiserver/pki/etcd-client-key.pem: /keys/etcd-client-key.pem
49 /etc/kubernetes/apiserver/pki/etcd-client.pem: /certs/etcd-client.pem
50 /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem: /certs/kubelet-client-ca.pem
51 /etc/kubernetes/apiserver/pki/kubelet-client-key.pem: /keys/kubelet-client-key.pem
52 /etc/kubernetes/apiserver/pki/kubelet-client.pem: /certs/kubelet-client.pem
53 /etc/kubernetes/apiserver/pki/service-account.pub: /certs/service-account.pub
54 /etc/kubernetes/manifests/kubernetes-apiserver.yaml: /tmp/etc/kubernetes-apiserver.yaml
55
17images: 56images:
18 tags: 57 tags:
19 anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.11 58 anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.11
@@ -30,65 +69,58 @@ anchor:
30 kubelet: 69 kubelet:
31 manifest_path: /etc/kubernetes/manifests 70 manifest_path: /etc/kubernetes/manifests
32 period: 15 71 period: 15
33 files_to_copy:
34 - source: /certs/apiserver.pem
35 dest: /etc/kubernetes/apiserver/pki/apiserver.pem
36 - source: /certs/kubelet-client.pem
37 dest: /etc/kubernetes/apiserver/pki/kubelet-client.pem
38 - source: /certs/kubelet-client-ca.pem
39 dest: /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
40 - source: /certs/cluster-ca.pem
41 dest: /etc/kubernetes/apiserver/pki/cluster-ca.pem
42 - source: /certs/etcd-client-ca.pem
43 dest: /etc/kubernetes/apiserver/pki/etcd-client-ca.pem
44 - source: /certs/etcd-client.pem
45 dest: /etc/kubernetes/apiserver/pki/etcd-client.pem
46 - source: /certs/service-account.pub
47 dest: /etc/kubernetes/apiserver/pki/service-account.pub
48 - source: /keys/apiserver-key.pem
49 dest: /etc/kubernetes/apiserver/pki/apiserver-key.pem
50 - source: /keys/kubelet-client-key.pem
51 dest: /etc/kubernetes/apiserver/pki/kubelet-client-key.pem
52 - source: /keys/etcd-client-key.pem
53 dest: /etc/kubernetes/apiserver/pki/etcd-client-key.pem
54 - source: /tmp/etc/kubernetes-apiserver.yaml
55 dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml
56 - source: /tmp/etc/kubeconfig.yaml
57 dest: /etc/kubernetes/apiserver/kubeconfig.yaml
58 # Note: config files for admission controllers are added to this dynamically
59 72
60command_prefix: 73conf:
61 - /apiserver 74# Uncomment any of the below to enable the file placement and associated apiserver
62 - --authorization-mode=Node,RBAC 75# command line options
63 - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit 76#
64 - --service-cluster-ip-range=10.96.0.0/16 77# acconfig:
65 - --endpoint-reconciler-type=lease 78# file: acconfig.yaml
66 # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 79# command_options:
67 - --repair-malformed-updates=false 80# - '--admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml'
81# - '--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit'
82# content:
83# kind: AdmissionConfiguration
84# apiVersion: apiserver.k8s.io/v1alpha1
85# plugins:
86# - name: EventRateLimit
87# path: eventconfig.yaml
88# eventconfig:
89# file: eventconfig.yaml
90# command_options:
91# - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml'
92# content:
93# kind: Configuration
94# apiVersion: eventratelimit.admission.k8s.io/v1alpha1
95# limits:
96# - type: Server
97# qps: 1000
98# burst: 10000
99# encryption_provider:
100# file: encryption_provider.yaml
101# command_option: ''
102# content:
103# kind: EncryptionConfig
104# apiVersion: v1
105# resources:
106# - resources:
107# - 'secrets'
108# providers:
109# - identity: {}
68 110
69apiserver: 111apiserver:
70 host_etc_path: /etc/kubernetes/apiserver 112 arguments:
113 - --authorization-mode=Node,RBAC
114 - --service-cluster-ip-range=10.96.0.0/16
115 - --endpoint-reconciler-type=lease
116 - --feature-gates=PodShareProcessNamespace=true
117 # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
118 - --repair-malformed-updates=false
119 - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction
120 - --v=3
71 etcd: 121 etcd:
72 endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local 122 endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local
73 123 host_etc_path: /etc/kubernetes/apiserver
74conf:
75 # Admission controllers config files are generated dynamically based on the
76 # config below, as they are specific to particular ACs that may be
77 # configured by the operator (or added by k8s in the future).
78 admission_controllers:
79 eventconfig.yaml:
80 kind: Configuration
81 apiVersion: eventratelimit.admission.k8s.io/v1alpha1
82 limits:
83 - type: Server
84 qps: 100
85 burst: 1000
86 acconfig.yaml:
87 kind: AdmissionConfiguration
88 apiVersion: apiserver.k8s.io/v1alpha1
89 plugins:
90 - name: EventRateLimit
91 path: eventconfig.yaml
92 124
93network: 125network:
94 kubernetes_apiserver: 126 kubernetes_apiserver:
@@ -130,7 +162,6 @@ secrets:
130 cert: null 162 cert: null
131 key: null 163 key: null
132 164
133
134# typically overriden by environmental 165# typically overriden by environmental
135# values, but should include all endpoints 166# values, but should include all endpoints
136# required by this chart 167# required by this chart
@@ -170,7 +201,7 @@ pod:
170 upgrades: 201 upgrades:
171 daemonsets: 202 daemonsets:
172 pod_replacement_strategy: RollingUpdate 203 pod_replacement_strategy: RollingUpdate
173 kubernetes_apiserver: 204 kubernetes-apiserver-anchor:
174 enabled: false 205 enabled: false
175 min_ready_seconds: 0 206 min_ready_seconds: 0
176 max_unavailable: 1 207 max_unavailable: 1
diff --git a/examples/basic/Genesis.yaml b/examples/basic/Genesis.yaml
index 823ae70..ddc1916 100644
--- a/examples/basic/Genesis.yaml
+++ b/examples/basic/Genesis.yaml
@@ -11,15 +11,16 @@ data:
11 hostname: n0 11 hostname: n0
12 ip: 192.168.77.10 12 ip: 192.168.77.10
13 apiserver: 13 apiserver:
14 command_prefix: 14 arguments:
15 - /apiserver
16 - --authorization-mode=Node,RBAC 15 - --authorization-mode=Node,RBAC
17 - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit 16 - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
18 - --service-cluster-ip-range=10.96.0.0/16 17 - --service-cluster-ip-range=10.96.0.0/16
19 - --endpoint-reconciler-type=lease 18 - --endpoint-reconciler-type=lease
20 - --feature-gates=PodShareProcessNamespace=true 19 - --feature-gates=PodShareProcessNamespace=true
21 # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 20 # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
22 - --repair-malformed-updates=false 21 - --repair-malformed-updates=false
22 - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
23 - --v=3
23 armada: 24 armada:
24 target_manifest: cluster-bootstrap 25 target_manifest: cluster-bootstrap
25 labels: 26 labels:
@@ -45,4 +46,22 @@ data:
45 - path: /var/lib/anchor/calico-etcd-bootstrap 46 - path: /var/lib/anchor/calico-etcd-bootstrap
46 content: "# placeholder for triggering calico etcd bootstrapping" 47 content: "# placeholder for triggering calico etcd bootstrapping"
47 mode: 0644 48 mode: 0644
49 # NOTE(mark-burnett): These are referenced by the apiserver arguments above.
50 - path: /etc/genesis/apiserver/acconfig.yaml
51 mode: 0444
52 content: |
53 kind: AdmissionConfiguration
54 apiVersion: apiserver.k8s.io/v1alpha1
55 plugins:
56 - name: EventRateLimit
57 path: eventconfig.yaml
58 - path: /etc/genesis/apiserver/eventconfig.yaml
59 mode: 0444
60 content: |
61 kind: Configuration
62 apiVersion: eventratelimit.admission.k8s.io/v1alpha1
63 limits:
64 - type: Server
65 qps: 1000
66 burst: 10000
48... 67...
diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml
index f99fdd3..8df50a1 100644
--- a/examples/basic/armada-resources.yaml
+++ b/examples/basic/armada-resources.yaml
@@ -719,15 +719,6 @@ data:
719 upgrade: 719 upgrade:
720 no_hooks: true 720 no_hooks: true
721 values: 721 values:
722 command_prefix:
723 - /apiserver
724 - --authorization-mode=Node,RBAC
725 - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
726 - --service-cluster-ip-range=10.96.0.0/16
727 - --endpoint-reconciler-type=lease
728 - --feature-gates=PodShareProcessNamespace=true
729 # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
730 - --repair-malformed-updates=false
731 apiserver: 722 apiserver:
732 etcd: 723 etcd:
733 endpoints: https://127.0.0.1:2378 724 endpoints: https://127.0.0.1:2378
diff --git a/promenade/config.py b/promenade/config.py
index 79febba..6553077 100644
--- a/promenade/config.py
+++ b/promenade/config.py
@@ -241,7 +241,7 @@ class Configuration:
241 241
242 def bootstrap_apiserver_prefix(self): 242 def bootstrap_apiserver_prefix(self):
243 return self.get_path('Genesis:apiserver.command_prefix', 243 return self.get_path('Genesis:apiserver.command_prefix',
244 ['/apiserver', '--apiserver-count=2', '--v=5']) 244 ['/apiserver'])
245 245
246 246
247def _matches_filter(document, *, schema, labels, name): 247def _matches_filter(document, *, schema, labels, name):
diff --git a/promenade/schemas/Genesis.yaml b/promenade/schemas/Genesis.yaml
index 12f9b5b..021f3c3 100644
--- a/promenade/schemas/Genesis.yaml
+++ b/promenade/schemas/Genesis.yaml
@@ -71,6 +71,10 @@ data:
71 type: array 71 type: array
72 items: 72 items:
73 type: string 73 type: string
74 arguments:
75 type: array
76 items:
77 type: string
74 additionalProperties: false 78 additionalProperties: false
75 79
76 files: 80 files:
diff --git a/promenade/templates/include/genesis-apiserver.yaml b/promenade/templates/include/genesis-apiserver.yaml
new file mode 100644
index 0000000..4314a61
--- /dev/null
+++ b/promenade/templates/include/genesis-apiserver.yaml
@@ -0,0 +1,18 @@
1 - --advertise-address={{ config['Genesis:ip'] }}
2 - --allow-privileged=true
3 - --anonymous-auth=false
4 - --bind-address=0.0.0.0
5 - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
6 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
7 - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
8 - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
9 - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
10 - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
11 - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
12 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
13 - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
14 - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
15 - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
16 {%- for argument in config.get_path('Genesis:apiserver.arguments', []) %}
17 - "{{ argument }}"
18 {%- endfor %}
diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml
deleted file mode 100644
index c792a8b..0000000
--- a/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
1---
2kind: AdmissionConfiguration
3apiVersion: apiserver.k8s.io/v1alpha1
4plugins:
5- name: EventRateLimit
6 path: eventconfig.yaml \ No newline at end of file
diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml
deleted file mode 100644
index ae78968..0000000
--- a/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
1---
2kind: Configuration
3apiVersion: eventratelimit.admission.k8s.io/v1alpha1
4limits:
5- type: Server
6 qps: 100
7 burst: 1000 \ No newline at end of file
diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
index d122d57..e9051aa 100644
--- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
+++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
@@ -11,146 +11,130 @@ spec:
11 dnsPolicy: Default 11 dnsPolicy: Default
12 hostNetwork: true 12 hostNetwork: true
13 containers: 13 containers:
14 - env: 14 - env:
15 - name: TILLER_NAMESPACE 15 - name: TILLER_NAMESPACE
16 value: kube-system 16 value: kube-system
17 image: {{ config['Genesis:images.helm.tiller'] }} 17 image: {{ config['Genesis:images.helm.tiller'] }}
18 command: 18 command:
19 - /tiller 19 - /tiller
20 - -logtostderr 20 - -logtostderr
21 - -v 21 - -v
22 - "99" 22 - "99"
23 imagePullPolicy: IfNotPresent 23 imagePullPolicy: IfNotPresent
24 livenessProbe: 24 livenessProbe:
25 failureThreshold: 3 25 failureThreshold: 3
26 httpGet: 26 httpGet:
27 path: /liveness 27 path: /liveness
28 port: 44135 28 port: 44135
29 scheme: HTTP 29 scheme: HTTP
30 initialDelaySeconds: 1 30 initialDelaySeconds: 1
31 periodSeconds: 10 31 periodSeconds: 10
32 successThreshold: 1 32 successThreshold: 1
33 timeoutSeconds: 1 33 timeoutSeconds: 1
34 name: tiller
35 ports:
36 - containerPort: 44134
37 name: tiller 34 name: tiller
38 protocol: TCP 35 ports:
39 readinessProbe: 36 - containerPort: 44134
40 failureThreshold: 3 37 name: tiller
41 httpGet: 38 protocol: TCP
42 path: /readiness 39 readinessProbe:
43 port: 44135 40 failureThreshold: 3
44 scheme: HTTP 41 httpGet:
45 initialDelaySeconds: 1 42 path: /readiness
46 periodSeconds: 10 43 port: 44135
47 successThreshold: 1 44 scheme: HTTP
48 timeoutSeconds: 1 45 initialDelaySeconds: 1
49 resources: {} 46 periodSeconds: 10
50 terminationMessagePath: /dev/termination-log 47 successThreshold: 1
51 terminationMessagePolicy: File 48 timeoutSeconds: 1
52 - name: armada 49 resources: {}
53 image: {{ config['Genesis:images.armada'] }} 50 terminationMessagePath: /dev/termination-log
54 securityContext: 51 terminationMessagePolicy: File
55 runAsUser: 0 52 - name: armada
56 command: 53 image: {{ config['Genesis:images.armada'] }}
57 - /bin/bash 54 securityContext:
58 - -c 55 runAsUser: 0
59 - |- 56 command:
60 set -x 57 - /bin/bash
58 - -c
59 - |-
60 set -x
61 61
62 while true; do 62 while true; do
63 sleep 10 63 sleep 10
64 if armada \ 64 if armada \
65 apply \ 65 apply \
66 --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \ 66 --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
67 --tiller-host 127.0.0.1 \ 67 --tiller-host 127.0.0.1 \
68 /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then 68 /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
69 break 69 break
70 fi 70 fi
71 done 71 done
72 touch /ipc/armada-done
73 sleep 10000
74 env:
75 - name: ARMADA_LOGFILE
76 value: /tmp/log/bootstrap-armada.log
77 {%- if config['KubernetesNetwork:proxy.url'] is defined %}
78 - name: HTTP_PROXY
79 value: {{ config['KubernetesNetwork:proxy.url'] }}
80 - name: HTTPS_PROXY
81 value: {{ config['KubernetesNetwork:proxy.url'] }}
82 - name: NO_PROXY
83 value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
84 - name: http_proxy
85 value: {{ config['KubernetesNetwork:proxy.url'] }}
86 - name: https_proxy
87 value: {{ config['KubernetesNetwork:proxy.url'] }}
88 - name: no_proxy
89 value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
90 {%- endif %}
91 volumeMounts:
92 - name: assets
93 mountPath: /etc/genesis/armada/assets
94 - name: auth
95 mountPath: /root/.kube
96 - name: ipc
97 mountPath: /ipc
98 - name: log
99 mountPath: /tmp/log
100 - name: monitor
101 image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
102 command:
103 - /bin/sh
104 - -c
105 - |-
106 set -x
72 107
73 touch /ipc/armada-done 108 while ! [ -e /ipc/armada-done ]; do
74 sleep 10000 109 sleep 5
75 env: 110 done
76 - name: ARMADA_LOGFILE
77 value: /tmp/log/bootstrap-armada.log
78{%- if config['KubernetesNetwork:proxy.url'] is defined %}
79 - name: HTTP_PROXY
80 value: {{ config['KubernetesNetwork:proxy.url'] }}
81 - name: HTTPS_PROXY
82 value: {{ config['KubernetesNetwork:proxy.url'] }}
83 - name: NO_PROXY
84 value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
85 - name: http_proxy
86 value: {{ config['KubernetesNetwork:proxy.url'] }}
87 - name: https_proxy
88 value: {{ config['KubernetesNetwork:proxy.url'] }}
89 - name: no_proxy
90 value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
91{%- endif %}
92 volumeMounts:
93 - name: assets
94 mountPath: /etc/genesis/armada/assets
95 - name: auth
96 mountPath: /root/.kube
97 - name: ipc
98 mountPath: /ipc
99 - name: log
100 mountPath: /tmp/log
101 - name: monitor
102 image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
103 command:
104 - /bin/sh
105 - -c
106 - |-
107 set -x
108 111
109 while ! [ -e /ipc/armada-done ]; do 112 rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml
110 sleep 5 113 sleep 10000
111 done 114 volumeMounts:
112 115 - name: ipc
113 rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml 116 mountPath: /ipc
114 sleep 10000 117 - name: manifest
115 volumeMounts: 118 mountPath: /etc/kubernetes/manifests
116 - name: ipc 119 - name: kubectl-apiserver
117 mountPath: /ipc 120 image: {{ config['Genesis:images.kubernetes.apiserver'] }}
118 - name: manifest 121 command:
119 mountPath: /etc/kubernetes/manifests 122 {%- for argument in config.bootstrap_apiserver_prefix() %}
120 - name: kubectl-apiserver 123 - "{{ argument }}"
121 image: {{ config['Genesis:images.kubernetes.apiserver'] }} 124 {%- endfor %}
122 command: 125{% include "genesis-apiserver.yaml" with context %}
123 {%- for argument in config.bootstrap_apiserver_prefix() %} 126 - --etcd-servers=https://localhost:12379
124 - "{{ argument }}" 127 - --insecure-port=8080
125 {%- endfor %} 128 - --secure-port=6444
126 - --advertise-address={{ config['Genesis:ip'] }} 129 env:
127 - --anonymous-auth=false 130 - name: KUBECONFIG
128 - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem 131 value: /etc/kubernetes/admin/config
129 - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem 132 volumeMounts:
130 - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem 133 - name: auth
131 - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem 134 mountPath: /etc/kubernetes/admin
132 - --insecure-port=8080 135 - name: config
133 - --secure-port=6444 136 mountPath: /etc/kubernetes/apiserver
134 - --bind-address=0.0.0.0 137 readOnly: true
135 - --allow-privileged=true
136 - --etcd-servers=https://localhost:12379
137 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
138 - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
139 - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
140 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
141 - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
142 - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
143 - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
144 - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
145 env:
146 - name: KUBECONFIG
147 value: /etc/kubernetes/admin/config
148 volumeMounts:
149 - name: auth
150 mountPath: /etc/kubernetes/admin
151 - name: config
152 mountPath: /etc/kubernetes/apiserver
153 readOnly: true
154 volumes: 138 volumes:
155 - name: assets 139 - name: assets
156 hostPath: 140 hostPath:
diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml
index 606f0f3..4113327 100644
--- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml
+++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml
@@ -19,25 +19,10 @@ spec:
19 {%- for argument in config.bootstrap_apiserver_prefix() %} 19 {%- for argument in config.bootstrap_apiserver_prefix() %}
20 - "{{ argument }}" 20 - "{{ argument }}"
21 {%- endfor %} 21 {%- endfor %}
22 - --advertise-address={{ config['Genesis:ip'] }} 22{% include "genesis-apiserver.yaml" with context %}
23 - --anonymous-auth=false 23 - --etcd-servers=https://localhost:2379
24 - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
25 - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
26 - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
27 - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
28 - --insecure-port=0 24 - --insecure-port=0
29 - --bind-address=0.0.0.0
30 - --secure-port=6443 25 - --secure-port=6443
31 - --allow-privileged=true
32 - --etcd-servers=https://localhost:2379
33 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
34 - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
35 - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
36 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
37 - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
38 - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
39 - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
40 - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
41 volumeMounts: 26 volumeMounts:
42 - name: config 27 - name: config
43 mountPath: /etc/kubernetes/apiserver 28 mountPath: /etc/kubernetes/apiserver