diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index 8771f123..d275954f 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -578,10 +578,11 @@ data: values: conf: anchor: - kubernetes_url: https://kubernetes.default:443 + kubernetes_url: https://10.96.0.1:443 services: - default: - kubernetes: + default: null + kube-system: + kubernetes-apiserver: server_opts: "check port 6443" conf_parts: frontend: @@ -591,7 +592,6 @@ data: - mode tcp - option tcp-check - option redispatch - kube-system: kubernetes-etcd: server_opts: "check port 2379" conf_parts: diff --git a/examples/complete/armada-resources.yaml b/examples/complete/armada-resources.yaml index 88ee01c2..3456df4f 100644 --- a/examples/complete/armada-resources.yaml +++ b/examples/complete/armada-resources.yaml @@ -618,10 +618,11 @@ data: values: conf: anchor: - kubernetes_url: https://kubernetes.default:443 + kubernetes_url: https://10.96.0.1:443 services: - default: - kubernetes: + default: null + kube-system: + kubernetes-apiserver: server_opts: "check port 6443" conf_parts: frontend: @@ -631,7 +632,6 @@ data: - mode tcp - option tcp-check - option redispatch - kube-system: kubernetes-etcd: server_opts: "check port 2379" conf_parts: diff --git a/promenade/config.py b/promenade/config.py index 4a771be0..f968a275 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -170,6 +170,10 @@ class Configuration: validation.check_schema(item) self.documents.append(item) + def bootstrap_apiserver_prefix(self): + return self.get_path('Genesis:apiserver.command_prefix', + ['/apiserver', '--apiserver-count=2', '--v=5']) + def _matches_filter(document, *, schema, labels): matches = True diff --git a/promenade/schemas/Genesis.yaml b/promenade/schemas/Genesis.yaml index f9286558..d2616983 100644 --- a/promenade/schemas/Genesis.yaml +++ b/promenade/schemas/Genesis.yaml @@ -64,6 +64,15 @@ data: type: string additionalProperties: false + apiserver: + type: object + properties: + command_prefix: + type: array + items: + type: string + additionalProperties: false + files: type: array items: diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml index 0b55efa3..d2f29fcc 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml @@ -118,8 +118,9 @@ spec: - name: kubectl-apiserver image: {{ config['Genesis:images.kubernetes.apiserver'] }} command: - - /hyperkube - - apiserver + {%- for argument in config.bootstrap_apiserver_prefix() %} + - "{{ argument }}" + {%- endfor %} - --advertise-address={{ config['Genesis:ip'] }} - --authorization-mode=Node,RBAC - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds @@ -128,9 +129,6 @@ spec: - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem - # Hard coding to 2 is a pretty safe move for now. This can be exposed - # with additional configuration later. - - --apiserver-count=2 - --insecure-port=8080 - --secure-port=6444 - --bind-address=0.0.0.0 @@ -145,7 +143,6 @@ spec: - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem - - --v=5 env: - name: KUBECONFIG value: /etc/kubernetes/admin/config diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml index e9b7bb8a..b0c43eeb 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml @@ -14,8 +14,9 @@ spec: - name: kube-apiserver image: {{ config['Genesis:images.kubernetes.apiserver'] }} command: - - /hyperkube - - apiserver + {%- for argument in config.bootstrap_apiserver_prefix() %} + - "{{ argument }}" + {%- endfor %} - --advertise-address={{ config['Genesis:ip'] }} - --authorization-mode=Node,RBAC - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds @@ -24,9 +25,6 @@ spec: - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem - # Hard coding 3 is a pretty safe move for now. This can be exposed - # with additional configuration later. - - --apiserver-count=3 - --insecure-port=0 - --bind-address=0.0.0.0 - --secure-port=6443 @@ -41,7 +39,6 @@ spec: - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem - - --v=5 volumeMounts: - name: config mountPath: /etc/kubernetes/apiserver diff --git a/tools/gate/config-templates/bootstrap-armada-config.yaml b/tools/gate/config-templates/bootstrap-armada-config.yaml index f4c88f2d..fc0b73e7 100644 --- a/tools/gate/config-templates/bootstrap-armada-config.yaml +++ b/tools/gate/config-templates/bootstrap-armada-config.yaml @@ -556,11 +556,12 @@ data: values: conf: anchor: - kubernetes_url: https://kubernetes.default:443 + kubernetes_url: https://10.96.0.1:443 services: - default: - kubernetes: - server_opts: "check" + default: null + kube-system: + kubernetes-apiserver: + server_opts: "check port 6443" conf_parts: frontend: - mode tcp @@ -569,7 +570,6 @@ data: - mode tcp - option tcp-check - option redispatch - kube-system: kubernetes-etcd: server_opts: "check" conf_parts: