summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--charts/apiserver/templates/bin/_anchor.tpl54
-rw-r--r--charts/apiserver/templates/configmap-etc.yaml29
-rw-r--r--charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl35
-rw-r--r--charts/apiserver/values.yaml143
-rw-r--r--examples/basic/Genesis.yaml25
-rw-r--r--examples/basic/armada-resources.yaml9
-rw-r--r--promenade/config.py2
-rw-r--r--promenade/schemas/Genesis.yaml4
-rw-r--r--promenade/templates/include/genesis-apiserver.yaml18
-rw-r--r--promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml6
-rw-r--r--promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml7
-rw-r--r--promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml256
-rw-r--r--promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml19
13 files changed, 317 insertions, 290 deletions
diff --git a/charts/apiserver/templates/bin/_anchor.tpl b/charts/apiserver/templates/bin/_anchor.tpl
index c311ffa..904a467 100644
--- a/charts/apiserver/templates/bin/_anchor.tpl
+++ b/charts/apiserver/templates/bin/_anchor.tpl
@@ -15,26 +15,54 @@
15 15
16set -x 16set -x
17 17
18compare_copy_files() { 18snapshot_files() {
19 SNAPSHOT_DIR=${1}
20 {{ range $dest, $source := .Values.const.files_to_copy }}
21 mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
22 cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
23 {{- end }}
24 {{ range $key, $val := .Values.conf }}
25 cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
26 {{- end }}
27}
19 28
20 {{range .Values.anchor.files_to_copy}} 29compare_copy_files() {
21 if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then 30 SNAPSHOT_DIR=${1}
22 mkdir -p $(dirname /host{{ .dest }}) 31 {{ range $dest, $source := .Values.const.files_to_copy }}
23 cp {{ .source }} /host{{ .dest }} 32 SRC="${SNAPSHOT_DIR}{{ $dest }}"
24 chmod go-rwx /host{{ .dest }} 33 DEST="/host{{ $dest }}"
34 if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
35 mkdir -p $(dirname "${DEST}")
36 cp "${SRC}" "${DEST}"
37 chmod go-rwx "${DEST}"
25 fi 38 fi
26 {{end}} 39 {{- end}}
40 {{ range $key, $val := .Values.conf }}
41 SRC="${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
42 DEST="/host/etc/kubernetes/apiserver/{{ $val.file }}"
43 if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
44 mkdir -p $(dirname "${DEST}")
45 cp "${SRC}" "${DEST}"
46 chmod go-rwx "${DEST}"
47 fi
48 {{- end }}
27} 49}
28 50
29cleanup() { 51cleanup() {
30 52 {{- range $dest, $source := .Values.const.files_to_copy }}
31 {{range .Values.anchor.files_to_copy}} 53 rm -f "/host{{ $dest }}"
32 rm -f /host{{ .dest }} 54 {{- end }}
33 {{end}} 55 {{ range $key, $val := .Values.conf }}
56 rm -f "/host/{{ $val.file }}"
57 {{- end }}
34} 58}
35 59
36while true; do
37 60
61SNAPSHOT_DIR=$(mktemp -d)
62
63snapshot_files "${SNAPSHOT_DIR}"
64
65while true; do
38 if [ -e /tmp/stop ]; then 66 if [ -e /tmp/stop ]; then
39 echo Stopping 67 echo Stopping
40 cleanup 68 cleanup
@@ -43,7 +71,7 @@ while true; do
43 71
44 # Compare and replace files on Genesis host if needed 72 # Compare and replace files on Genesis host if needed
45 # Copy files to other master nodes 73 # Copy files to other master nodes
46 compare_copy_files 74 compare_copy_files "${SNAPSHOT_DIR}"
47 75
48 sleep {{ .Values.anchor.period }} 76 sleep {{ .Values.anchor.period }}
49done 77done
diff --git a/charts/apiserver/templates/configmap-etc.yaml b/charts/apiserver/templates/configmap-etc.yaml
index 75a22ea..016290f 100644
--- a/charts/apiserver/templates/configmap-etc.yaml
+++ b/charts/apiserver/templates/configmap-etc.yaml
@@ -17,34 +17,19 @@ limitations under the License.
17{{- if .Values.manifests.configmap_etc }} 17{{- if .Values.manifests.configmap_etc }}
18{{- $envAll := . }} 18{{- $envAll := . }}
19 19
20{{/* This slightly involved merge of AC config files into the anchor
21 files uses HTK merge, as straighforward appends result in duplicates. */}}
22{{- $_ := set .Values "_ac_files_to_copy" list }}
23{{- range $key, $val := .Values.conf.admission_controllers }}
24 {{- $source := printf "/tmp/etc/%s" $key }}
25 {{- $dest := printf "/etc/kubernetes/apiserver/%s" $key }}
26 {{- $file_to_copy := dict "source" $source "dest" $dest }}
27 {{- $ac_files_to_copy := append $.Values._ac_files_to_copy $file_to_copy }}
28 {{- $_ := set $.Values "_ac_files_to_copy" $ac_files_to_copy }}
29{{- end }}
30{{ $all_files_to_copy := dict }}
31{{ $_ := set $all_files_to_copy "values" (tuple .Values.anchor.files_to_copy .Values._ac_files_to_copy) }}
32{{ $_ := $all_files_to_copy | include "helm-toolkit.utils.merge" }}
33{{ $_ := set .Values.anchor "files_to_copy" $all_files_to_copy.result }}
34
35--- 20---
36apiVersion: v1 21apiVersion: v1
37kind: ConfigMap 22kind: ConfigMap
38metadata: 23metadata:
39 name: {{ .Values.service.name }}-etc 24 name: {{ .Values.service.name }}-etc
40data: 25data:
41 kubernetes-apiserver.yaml: |+ 26 kubernetes-apiserver.yaml: |
42{{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} 27{{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
43 kubeconfig.yaml: |+ 28 kubeconfig.yaml: |
44{{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} 29{{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
45{{/* Dynamically add config files for admission controllers */}} 30{{/* Dynamically added config files */}}
46{{ range $key, $val := .Values.conf.admission_controllers }} 31{{- range $key, $val := .Values.conf }}
47 {{ $key }}: |+ 32 {{ $val.file }}: |
48{{ toYaml $val | indent 4 }} 33{{ toYaml $val.content | indent 4 }}
49{{ end }} 34{{- end }}
50{{- end }} 35{{- end }}
diff --git a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
index daf04e1..73f6ccf 100644
--- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
+++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
@@ -42,30 +42,25 @@ spec:
42 fieldPath: spec.nodeName 42 fieldPath: spec.nodeName
43 - name: KUBECONFIG 43 - name: KUBECONFIG
44 value: /etc/kubernetes/apiserver/kubeconfig.yaml 44 value: /etc/kubernetes/apiserver/kubeconfig.yaml
45 - name: APISERVER_PORT
46 value: {{ .Values.network.kubernetes_apiserver.port | quote }}
47 - name: ETCD_ENDPOINTS
48 value: {{ .Values.apiserver.etcd.endpoints | quote }}
45 49
46 command: 50 command:
47 {{- range .Values.command_prefix }} 51 {{- range .Values.const.command_prefix }}
48 - {{ . }} 52 - {{ . }}
49 {{- end }} 53 {{- end }}
50 - --advertise-address=$(POD_IP) 54 {{- range .Values.apiserver.arguments }}
51 - --anonymous-auth=false 55 - {{ . }}
52 - --bind-address=0.0.0.0 56 {{- end }}
53 - --secure-port={{ .Values.network.kubernetes_apiserver.port }} 57 {{- range $key, $val := .Values.conf }}
54 - --insecure-port=0 58 {{- if hasKey $val "command_options" }}
55 - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem 59 {{- range $val.command_options }}
56 - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem 60 - {{ . }}
57 - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem 61 {{- end }}
58 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname 62 {{- end }}
59 - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem 63 {{- end }}
60 - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
61 - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
62 - --etcd-servers={{ .Values.apiserver.etcd.endpoints }}
63 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
64 - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
65 - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
66 - --allow-privileged=true
67 - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
68 - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
69 64
70 ports: 65 ports:
71 - containerPort: {{ .Values.network.kubernetes_apiserver.port }} 66 - containerPort: {{ .Values.network.kubernetes_apiserver.port }}
diff --git a/charts/apiserver/values.yaml b/charts/apiserver/values.yaml
index 9c0556b..b7c5ecf 100644
--- a/charts/apiserver/values.yaml
+++ b/charts/apiserver/values.yaml
@@ -14,6 +14,45 @@
14 14
15release_group: null 15release_group: null
16 16
17# NOTE(mark-burnett): These values are not really configurable -- they live
18# here to keep the templates cleaner.
19const:
20 command_prefix:
21 - /apiserver
22 - --advertise-address=$(POD_IP)
23 - --allow-privileged=true
24 - --anonymous-auth=false
25 - --bind-address=0.0.0.0
26 - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
27 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
28 - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
29 - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
30 - --etcd-servers=$(ETCD_ENDPOINTS)
31 - --insecure-port=0
32 - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
33 - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
34 - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
35 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
36 - --secure-port=$(APISERVER_PORT)
37 - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
38 - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
39 - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
40
41 files_to_copy:
42 # NOTE(mark-burnett): These are (host dest): (container source) pairs
43 /etc/kubernetes/apiserver/kubeconfig.yaml: /tmp/etc/kubeconfig.yaml
44 /etc/kubernetes/apiserver/pki/apiserver-key.pem: /keys/apiserver-key.pem
45 /etc/kubernetes/apiserver/pki/apiserver.pem: /certs/apiserver.pem
46 /etc/kubernetes/apiserver/pki/cluster-ca.pem: /certs/cluster-ca.pem
47 /etc/kubernetes/apiserver/pki/etcd-client-ca.pem: /certs/etcd-client-ca.pem
48 /etc/kubernetes/apiserver/pki/etcd-client-key.pem: /keys/etcd-client-key.pem
49 /etc/kubernetes/apiserver/pki/etcd-client.pem: /certs/etcd-client.pem
50 /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem: /certs/kubelet-client-ca.pem
51 /etc/kubernetes/apiserver/pki/kubelet-client-key.pem: /keys/kubelet-client-key.pem
52 /etc/kubernetes/apiserver/pki/kubelet-client.pem: /certs/kubelet-client.pem
53 /etc/kubernetes/apiserver/pki/service-account.pub: /certs/service-account.pub
54 /etc/kubernetes/manifests/kubernetes-apiserver.yaml: /tmp/etc/kubernetes-apiserver.yaml
55
17images: 56images:
18 tags: 57 tags:
19 anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.11 58 anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.11
@@ -30,65 +69,58 @@ anchor:
30 kubelet: 69 kubelet:
31 manifest_path: /etc/kubernetes/manifests 70 manifest_path: /etc/kubernetes/manifests
32 period: 15 71 period: 15
33 files_to_copy:
34 - source: /certs/apiserver.pem
35 dest: /etc/kubernetes/apiserver/pki/apiserver.pem
36 - source: /certs/kubelet-client.pem
37 dest: /etc/kubernetes/apiserver/pki/kubelet-client.pem
38 - source: /certs/kubelet-client-ca.pem
39 dest: /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
40 - source: /certs/cluster-ca.pem
41 dest: /etc/kubernetes/apiserver/pki/cluster-ca.pem
42 - source: /certs/etcd-client-ca.pem
43 dest: /etc/kubernetes/apiserver/pki/etcd-client-ca.pem
44 - source: /certs/etcd-client.pem
45 dest: /etc/kubernetes/apiserver/pki/etcd-client.pem
46 - source: /certs/service-account.pub
47 dest: /etc/kubernetes/apiserver/pki/service-account.pub
48 - source: /keys/apiserver-key.pem
49 dest: /etc/kubernetes/apiserver/pki/apiserver-key.pem
50 - source: /keys/kubelet-client-key.pem
51 dest: /etc/kubernetes/apiserver/pki/kubelet-client-key.pem
52 - source: /keys/etcd-client-key.pem
53 dest: /etc/kubernetes/apiserver/pki/etcd-client-key.pem
54 - source: /tmp/etc/kubernetes-apiserver.yaml
55 dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml
56 - source: /tmp/etc/kubeconfig.yaml
57 dest: /etc/kubernetes/apiserver/kubeconfig.yaml
58 # Note: config files for admission controllers are added to this dynamically
59 72
60command_prefix: 73conf:
61 - /apiserver 74# Uncomment any of the below to enable the file placement and associated apiserver
62 - --authorization-mode=Node,RBAC 75# command line options
63 - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit 76#
64 - --service-cluster-ip-range=10.96.0.0/16 77# acconfig:
65 - --endpoint-reconciler-type=lease 78# file: acconfig.yaml
66 # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 79# command_options:
67 - --repair-malformed-updates=false 80# - '--admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml'
81# - '--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit'
82# content:
83# kind: AdmissionConfiguration
84# apiVersion: apiserver.k8s.io/v1alpha1
85# plugins:
86# - name: EventRateLimit
87# path: eventconfig.yaml
88# eventconfig:
89# file: eventconfig.yaml
90# command_options:
91# - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml'
92# content:
93# kind: Configuration
94# apiVersion: eventratelimit.admission.k8s.io/v1alpha1
95# limits:
96# - type: Server
97# qps: 1000
98# burst: 10000
99# encryption_provider:
100# file: encryption_provider.yaml
101# command_option: ''
102# content:
103# kind: EncryptionConfig
104# apiVersion: v1
105# resources:
106# - resources:
107# - 'secrets'
108# providers:
109# - identity: {}
68 110
69apiserver: 111apiserver:
70 host_etc_path: /etc/kubernetes/apiserver 112 arguments:
113 - --authorization-mode=Node,RBAC
114 - --service-cluster-ip-range=10.96.0.0/16
115 - --endpoint-reconciler-type=lease
116 - --feature-gates=PodShareProcessNamespace=true
117 # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
118 - --repair-malformed-updates=false
119 - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction
120 - --v=3
71 etcd: 121 etcd:
72 endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local 122 endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local
73 123 host_etc_path: /etc/kubernetes/apiserver
74conf:
75 # Admission controllers config files are generated dynamically based on the
76 # config below, as they are specific to particular ACs that may be
77 # configured by the operator (or added by k8s in the future).
78 admission_controllers:
79 eventconfig.yaml:
80 kind: Configuration
81 apiVersion: eventratelimit.admission.k8s.io/v1alpha1
82 limits:
83 - type: Server
84 qps: 100
85 burst: 1000
86 acconfig.yaml:
87 kind: AdmissionConfiguration
88 apiVersion: apiserver.k8s.io/v1alpha1
89 plugins:
90 - name: EventRateLimit
91 path: eventconfig.yaml
92 124
93network: 125network:
94 kubernetes_apiserver: 126 kubernetes_apiserver:
@@ -130,7 +162,6 @@ secrets:
130 cert: null 162 cert: null
131 key: null 163 key: null
132 164
133
134# typically overriden by environmental 165# typically overriden by environmental
135# values, but should include all endpoints 166# values, but should include all endpoints
136# required by this chart 167# required by this chart
@@ -170,7 +201,7 @@ pod:
170 upgrades: 201 upgrades:
171 daemonsets: 202 daemonsets:
172 pod_replacement_strategy: RollingUpdate 203 pod_replacement_strategy: RollingUpdate
173 kubernetes_apiserver: 204 kubernetes-apiserver-anchor:
174 enabled: false 205 enabled: false
175 min_ready_seconds: 0 206 min_ready_seconds: 0
176 max_unavailable: 1 207 max_unavailable: 1
diff --git a/examples/basic/Genesis.yaml b/examples/basic/Genesis.yaml
index 823ae70..ddc1916 100644
--- a/examples/basic/Genesis.yaml
+++ b/examples/basic/Genesis.yaml
@@ -11,15 +11,16 @@ data:
11 hostname: n0 11 hostname: n0
12 ip: 192.168.77.10 12 ip: 192.168.77.10
13 apiserver: 13 apiserver:
14 command_prefix: 14 arguments:
15 - /apiserver
16 - --authorization-mode=Node,RBAC 15 - --authorization-mode=Node,RBAC
17 - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit 16 - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
18 - --service-cluster-ip-range=10.96.0.0/16 17 - --service-cluster-ip-range=10.96.0.0/16
19 - --endpoint-reconciler-type=lease 18 - --endpoint-reconciler-type=lease
20 - --feature-gates=PodShareProcessNamespace=true 19 - --feature-gates=PodShareProcessNamespace=true
21 # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 20 # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
22 - --repair-malformed-updates=false 21 - --repair-malformed-updates=false
22 - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
23 - --v=3
23 armada: 24 armada:
24 target_manifest: cluster-bootstrap 25 target_manifest: cluster-bootstrap
25 labels: 26 labels:
@@ -45,4 +46,22 @@ data:
45 - path: /var/lib/anchor/calico-etcd-bootstrap 46 - path: /var/lib/anchor/calico-etcd-bootstrap
46 content: "# placeholder for triggering calico etcd bootstrapping" 47 content: "# placeholder for triggering calico etcd bootstrapping"
47 mode: 0644 48 mode: 0644
49 # NOTE(mark-burnett): These are referenced by the apiserver arguments above.
50 - path: /etc/genesis/apiserver/acconfig.yaml
51 mode: 0444
52 content: |
53 kind: AdmissionConfiguration
54 apiVersion: apiserver.k8s.io/v1alpha1
55 plugins:
56 - name: EventRateLimit
57 path: eventconfig.yaml
58 - path: /etc/genesis/apiserver/eventconfig.yaml
59 mode: 0444
60 content: |
61 kind: Configuration
62 apiVersion: eventratelimit.admission.k8s.io/v1alpha1
63 limits:
64 - type: Server
65 qps: 1000
66 burst: 10000
48... 67...
diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml
index f99fdd3..8df50a1 100644
--- a/examples/basic/armada-resources.yaml
+++ b/examples/basic/armada-resources.yaml
@@ -719,15 +719,6 @@ data:
719 upgrade: 719 upgrade:
720 no_hooks: true 720 no_hooks: true
721 values: 721 values:
722 command_prefix:
723 - /apiserver
724 - --authorization-mode=Node,RBAC
725 - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
726 - --service-cluster-ip-range=10.96.0.0/16
727 - --endpoint-reconciler-type=lease
728 - --feature-gates=PodShareProcessNamespace=true
729 # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
730 - --repair-malformed-updates=false
731 apiserver: 722 apiserver:
732 etcd: 723 etcd:
733 endpoints: https://127.0.0.1:2378 724 endpoints: https://127.0.0.1:2378
diff --git a/promenade/config.py b/promenade/config.py
index 79febba..6553077 100644
--- a/promenade/config.py
+++ b/promenade/config.py
@@ -241,7 +241,7 @@ class Configuration:
241 241
242 def bootstrap_apiserver_prefix(self): 242 def bootstrap_apiserver_prefix(self):
243 return self.get_path('Genesis:apiserver.command_prefix', 243 return self.get_path('Genesis:apiserver.command_prefix',
244 ['/apiserver', '--apiserver-count=2', '--v=5']) 244 ['/apiserver'])
245 245
246 246
247def _matches_filter(document, *, schema, labels, name): 247def _matches_filter(document, *, schema, labels, name):
diff --git a/promenade/schemas/Genesis.yaml b/promenade/schemas/Genesis.yaml
index 12f9b5b..021f3c3 100644
--- a/promenade/schemas/Genesis.yaml
+++ b/promenade/schemas/Genesis.yaml
@@ -71,6 +71,10 @@ data:
71 type: array 71 type: array
72 items: 72 items:
73 type: string 73 type: string
74 arguments:
75 type: array
76 items:
77 type: string
74 additionalProperties: false 78 additionalProperties: false
75 79
76 files: 80 files:
diff --git a/promenade/templates/include/genesis-apiserver.yaml b/promenade/templates/include/genesis-apiserver.yaml
new file mode 100644
index 0000000..4314a61
--- /dev/null
+++ b/promenade/templates/include/genesis-apiserver.yaml
@@ -0,0 +1,18 @@
1 - --advertise-address={{ config['Genesis:ip'] }}
2 - --allow-privileged=true
3 - --anonymous-auth=false
4 - --bind-address=0.0.0.0
5 - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
6 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
7 - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
8 - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
9 - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
10 - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
11 - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
12 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
13 - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
14 - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
15 - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
16 {%- for argument in config.get_path('Genesis:apiserver.arguments', []) %}
17 - "{{ argument }}"
18 {%- endfor %}
diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml
deleted file mode 100644
index c792a8b..0000000
--- a/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
1---
2kind: AdmissionConfiguration
3apiVersion: apiserver.k8s.io/v1alpha1
4plugins:
5- name: EventRateLimit
6 path: eventconfig.yaml \ No newline at end of file
diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml
deleted file mode 100644
index ae78968..0000000
--- a/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
1---
2kind: Configuration
3apiVersion: eventratelimit.admission.k8s.io/v1alpha1
4limits:
5- type: Server
6 qps: 100
7 burst: 1000 \ No newline at end of file
diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
index d122d57..e9051aa 100644
--- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
+++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
@@ -11,146 +11,130 @@ spec:
11 dnsPolicy: Default 11 dnsPolicy: Default
12 hostNetwork: true 12 hostNetwork: true
13 containers: 13 containers:
14 - env: 14 - env:
15 - name: TILLER_NAMESPACE 15 - name: TILLER_NAMESPACE
16 value: kube-system 16 value: kube-system
17 image: {{ config['Genesis:images.helm.tiller'] }} 17 image: {{ config['Genesis:images.helm.tiller'] }}
18 command: 18 command:
19 - /tiller 19 - /tiller
20 - -logtostderr 20 - -logtostderr
21 - -v 21 - -v
22 - "99" 22 - "99"
23 imagePullPolicy: IfNotPresent 23 imagePullPolicy: IfNotPresent
24 livenessProbe: 24 livenessProbe:
25 failureThreshold: 3 25 failureThreshold: 3
26 httpGet: 26 httpGet:
27 path: /liveness 27 path: /liveness
28 port: 44135 28 port: 44135
29 scheme: HTTP 29 scheme: HTTP
30 initialDelaySeconds: 1 30 initialDelaySeconds: 1
31 periodSeconds: 10 31 periodSeconds: 10
32 successThreshold: 1 32 successThreshold: 1
33 timeoutSeconds: 1 33 timeoutSeconds: 1
34 name: tiller
35 ports:
36 - containerPort: 44134
37 name: tiller 34 name: tiller
38 protocol: TCP 35 ports:
39 readinessProbe: 36 - containerPort: 44134
40 failureThreshold: 3 37 name: tiller
41 httpGet: 38 protocol: TCP
42 path: /readiness 39 readinessProbe:
43 port: 44135 40 failureThreshold: 3
44 scheme: HTTP 41 httpGet:
45 initialDelaySeconds: 1 42 path: /readiness
46 periodSeconds: 10 43 port: 44135
47 successThreshold: 1 44 scheme: HTTP
48 timeoutSeconds: 1 45 initialDelaySeconds: 1
49 resources: {} 46 periodSeconds: 10
50 terminationMessagePath: /dev/termination-log 47 successThreshold: 1
51 terminationMessagePolicy: File 48 timeoutSeconds: 1
52 - name: armada 49 resources: {}
53 image: {{ config['Genesis:images.armada'] }} 50 terminationMessagePath: /dev/termination-log
54 securityContext: 51 terminationMessagePolicy: File
55 runAsUser: 0 52 - name: armada
56 command: 53 image: {{ config['Genesis:images.armada'] }}
57 - /bin/bash 54 securityContext:
58 - -c 55 runAsUser: 0
59 - |- 56 command:
60 set -x 57 - /bin/bash
58 - -c
59 - |-
60 set -x
61 61
62 while true; do 62 while true; do
63 sleep 10 63 sleep 10
64 if armada \ 64 if armada \
65 apply \ 65 apply \
66 --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \ 66 --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
67 --tiller-host 127.0.0.1 \ 67 --tiller-host 127.0.0.1 \
68 /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then 68 /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
69 break 69 break
70 fi 70 fi
71 done 71 done
72 touch /ipc/armada-done
73 sleep 10000
74 env:
75 - name: ARMADA_LOGFILE
76 value: /tmp/log/bootstrap-armada.log
77 {%- if config['KubernetesNetwork:proxy.url'] is defined %}
78 - name: HTTP_PROXY
79 value: {{ config['KubernetesNetwork:proxy.url'] }}
80 - name: HTTPS_PROXY
81 value: {{ config['KubernetesNetwork:proxy.url'] }}
82 - name: NO_PROXY
83 value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
84 - name: http_proxy
85 value: {{ config['KubernetesNetwork:proxy.url'] }}
86 - name: https_proxy
87 value: {{ config['KubernetesNetwork:proxy.url'] }}
88 - name: no_proxy
89 value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
90 {%- endif %}
91 volumeMounts:
92 - name: assets
93 mountPath: /etc/genesis/armada/assets
94 - name: auth
95 mountPath: /root/.kube
96 - name: ipc
97 mountPath: /ipc
98 - name: log
99 mountPath: /tmp/log
100 - name: monitor
101 image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
102 command:
103 - /bin/sh
104 - -c
105 - |-
106 set -x
72 107
73 touch /ipc/armada-done 108 while ! [ -e /ipc/armada-done ]; do
74 sleep 10000 109 sleep 5
75 env: 110 done
76 - name: ARMADA_LOGFILE
77 value: /tmp/log/bootstrap-armada.log
78{%- if config['KubernetesNetwork:proxy.url'] is defined %}
79 - name: HTTP_PROXY
80 value: {{ config['KubernetesNetwork:proxy.url'] }}
81 - name: HTTPS_PROXY
82 value: {{ config['KubernetesNetwork:proxy.url'] }}
83 - name: NO_PROXY
84 value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
85 - name: http_proxy
86 value: {{ config['KubernetesNetwork:proxy.url'] }}
87 - name: https_proxy
88 value: {{ config['KubernetesNetwork:proxy.url'] }}
89 - name: no_proxy
90 value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}
91{%- endif %}
92 volumeMounts:
93 - name: assets
94 mountPath: /etc/genesis/armada/assets
95 - name: auth
96 mountPath: /root/.kube
97 - name: ipc
98 mountPath: /ipc
99 - name: log
100 mountPath: /tmp/log
101 - name: monitor
102 image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
103 command:
104 - /bin/sh
105 - -c
106 - |-
107 set -x
108 111
109 while ! [ -e /ipc/armada-done ]; do 112 rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml
110 sleep 5 113 sleep 10000
111 done 114 volumeMounts:
112 115 - name: ipc
113 rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml 116 mountPath: /ipc
114 sleep 10000 117 - name: manifest
115 volumeMounts: 118 mountPath: /etc/kubernetes/manifests
116 - name: ipc 119 - name: kubectl-apiserver
117 mountPath: /ipc 120 image: {{ config['Genesis:images.kubernetes.apiserver'] }}
118 - name: manifest 121 command:
119 mountPath: /etc/kubernetes/manifests 122 {%- for argument in config.bootstrap_apiserver_prefix() %}
120 - name: kubectl-apiserver 123 - "{{ argument }}"
121 image: {{ config['Genesis:images.kubernetes.apiserver'] }} 124 {%- endfor %}
122 command: 125{% include "genesis-apiserver.yaml" with context %}
123 {%- for argument in config.bootstrap_apiserver_prefix() %} 126 - --etcd-servers=https://localhost:12379
124 - "{{ argument }}" 127 - --insecure-port=8080
125 {%- endfor %} 128 - --secure-port=6444
126 - --advertise-address={{ config['Genesis:ip'] }} 129 env:
127 - --anonymous-auth=false 130 - name: KUBECONFIG
128 - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem 131 value: /etc/kubernetes/admin/config
129 - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem 132 volumeMounts:
130 - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem 133 - name: auth
131 - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem 134 mountPath: /etc/kubernetes/admin
132 - --insecure-port=8080 135 - name: config
133 - --secure-port=6444 136 mountPath: /etc/kubernetes/apiserver
134 - --bind-address=0.0.0.0 137 readOnly: true
135 - --allow-privileged=true
136 - --etcd-servers=https://localhost:12379
137 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
138 - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
139 - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
140 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
141 - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
142 - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
143 - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
144 - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
145 env:
146 - name: KUBECONFIG
147 value: /etc/kubernetes/admin/config
148 volumeMounts:
149 - name: auth
150 mountPath: /etc/kubernetes/admin
151 - name: config
152 mountPath: /etc/kubernetes/apiserver
153 readOnly: true
154 volumes: 138 volumes:
155 - name: assets 139 - name: assets
156 hostPath: 140 hostPath:
diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml
index 606f0f3..4113327 100644
--- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml
+++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml
@@ -19,25 +19,10 @@ spec:
19 {%- for argument in config.bootstrap_apiserver_prefix() %} 19 {%- for argument in config.bootstrap_apiserver_prefix() %}
20 - "{{ argument }}" 20 - "{{ argument }}"
21 {%- endfor %} 21 {%- endfor %}
22 - --advertise-address={{ config['Genesis:ip'] }} 22{% include "genesis-apiserver.yaml" with context %}
23 - --anonymous-auth=false 23 - --etcd-servers=https://localhost:2379
24 - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
25 - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
26 - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
27 - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
28 - --insecure-port=0 24 - --insecure-port=0
29 - --bind-address=0.0.0.0
30 - --secure-port=6443 25 - --secure-port=6443
31 - --allow-privileged=true
32 - --etcd-servers=https://localhost:2379
33 - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
34 - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
35 - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
36 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
37 - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
38 - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
39 - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
40 - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
41 volumeMounts: 26 volumeMounts:
42 - name: config 27 - name: config
43 mountPath: /etc/kubernetes/apiserver 28 mountPath: /etc/kubernetes/apiserver