From 4f975a8cd80ca65506c5cffb43e3aa8473ee4639 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Mon, 16 Apr 2018 09:25:45 -0500 Subject: [PATCH] Allow configuration of bootstrap API server This avoids possible issues when the configuration of the bootstrapping apiserver differs from the chart's configuration. Issues were specifically seen when overriding the node port range, but this opens up additional configuration also. Change-Id: I2a3fc5847e850c8055c099bac50782debbbabbf4 --- examples/basic/armada-resources.yaml | 8 ++++---- examples/complete/armada-resources.yaml | 8 ++++---- promenade/config.py | 4 ++++ promenade/schemas/Genesis.yaml | 9 +++++++++ .../etc/kubernetes/manifests/bootstrap-armada.yaml | 9 +++------ .../etc/kubernetes/manifests/kubernetes-apiserver.yaml | 9 +++------ .../gate/config-templates/bootstrap-armada-config.yaml | 10 +++++----- 7 files changed, 32 insertions(+), 25 deletions(-) diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index 8771f123..d275954f 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -578,10 +578,11 @@ data: values: conf: anchor: - kubernetes_url: https://kubernetes.default:443 + kubernetes_url: https://10.96.0.1:443 services: - default: - kubernetes: + default: null + kube-system: + kubernetes-apiserver: server_opts: "check port 6443" conf_parts: frontend: @@ -591,7 +592,6 @@ data: - mode tcp - option tcp-check - option redispatch - kube-system: kubernetes-etcd: server_opts: "check port 2379" conf_parts: diff --git a/examples/complete/armada-resources.yaml b/examples/complete/armada-resources.yaml index 88ee01c2..3456df4f 100644 --- a/examples/complete/armada-resources.yaml +++ b/examples/complete/armada-resources.yaml @@ -618,10 +618,11 @@ data: values: conf: anchor: - kubernetes_url: https://kubernetes.default:443 + kubernetes_url: https://10.96.0.1:443 services: - default: - kubernetes: + default: null + kube-system: + kubernetes-apiserver: server_opts: "check port 6443" conf_parts: frontend: @@ -631,7 +632,6 @@ data: - mode tcp - option tcp-check - option redispatch - kube-system: kubernetes-etcd: server_opts: "check port 2379" conf_parts: diff --git a/promenade/config.py b/promenade/config.py index 4a771be0..f968a275 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -170,6 +170,10 @@ class Configuration: validation.check_schema(item) self.documents.append(item) + def bootstrap_apiserver_prefix(self): + return self.get_path('Genesis:apiserver.command_prefix', + ['/apiserver', '--apiserver-count=2', '--v=5']) + def _matches_filter(document, *, schema, labels): matches = True diff --git a/promenade/schemas/Genesis.yaml b/promenade/schemas/Genesis.yaml index f9286558..d2616983 100644 --- a/promenade/schemas/Genesis.yaml +++ b/promenade/schemas/Genesis.yaml @@ -64,6 +64,15 @@ data: type: string additionalProperties: false + apiserver: + type: object + properties: + command_prefix: + type: array + items: + type: string + additionalProperties: false + files: type: array items: diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml index 0b55efa3..d2f29fcc 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml @@ -118,8 +118,9 @@ spec: - name: kubectl-apiserver image: {{ config['Genesis:images.kubernetes.apiserver'] }} command: - - /hyperkube - - apiserver + {%- for argument in config.bootstrap_apiserver_prefix() %} + - "{{ argument }}" + {%- endfor %} - --advertise-address={{ config['Genesis:ip'] }} - --authorization-mode=Node,RBAC - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds @@ -128,9 +129,6 @@ spec: - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem - # Hard coding to 2 is a pretty safe move for now. This can be exposed - # with additional configuration later. - - --apiserver-count=2 - --insecure-port=8080 - --secure-port=6444 - --bind-address=0.0.0.0 @@ -145,7 +143,6 @@ spec: - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem - - --v=5 env: - name: KUBECONFIG value: /etc/kubernetes/admin/config diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml index e9b7bb8a..b0c43eeb 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml @@ -14,8 +14,9 @@ spec: - name: kube-apiserver image: {{ config['Genesis:images.kubernetes.apiserver'] }} command: - - /hyperkube - - apiserver + {%- for argument in config.bootstrap_apiserver_prefix() %} + - "{{ argument }}" + {%- endfor %} - --advertise-address={{ config['Genesis:ip'] }} - --authorization-mode=Node,RBAC - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds @@ -24,9 +25,6 @@ spec: - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem - # Hard coding 3 is a pretty safe move for now. This can be exposed - # with additional configuration later. - - --apiserver-count=3 - --insecure-port=0 - --bind-address=0.0.0.0 - --secure-port=6443 @@ -41,7 +39,6 @@ spec: - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem - - --v=5 volumeMounts: - name: config mountPath: /etc/kubernetes/apiserver diff --git a/tools/gate/config-templates/bootstrap-armada-config.yaml b/tools/gate/config-templates/bootstrap-armada-config.yaml index f4c88f2d..fc0b73e7 100644 --- a/tools/gate/config-templates/bootstrap-armada-config.yaml +++ b/tools/gate/config-templates/bootstrap-armada-config.yaml @@ -556,11 +556,12 @@ data: values: conf: anchor: - kubernetes_url: https://kubernetes.default:443 + kubernetes_url: https://10.96.0.1:443 services: - default: - kubernetes: - server_opts: "check" + default: null + kube-system: + kubernetes-apiserver: + server_opts: "check port 6443" conf_parts: frontend: - mode tcp @@ -569,7 +570,6 @@ data: - mode tcp - option tcp-check - option redispatch - kube-system: kubernetes-etcd: server_opts: "check" conf_parts: