diff --git a/charts/apiserver/templates/bin/_anchor.tpl b/charts/apiserver/templates/bin/_anchor.tpl index c311ffa0..904a4670 100644 --- a/charts/apiserver/templates/bin/_anchor.tpl +++ b/charts/apiserver/templates/bin/_anchor.tpl @@ -15,26 +15,54 @@ set -x -compare_copy_files() { +snapshot_files() { + SNAPSHOT_DIR=${1} + {{ range $dest, $source := .Values.const.files_to_copy }} + mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}") + cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}" + {{- end }} + {{ range $key, $val := .Values.conf }} + cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}" + {{- end }} +} - {{range .Values.anchor.files_to_copy}} - if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then - mkdir -p $(dirname /host{{ .dest }}) - cp {{ .source }} /host{{ .dest }} - chmod go-rwx /host{{ .dest }} +compare_copy_files() { + SNAPSHOT_DIR=${1} + {{ range $dest, $source := .Values.const.files_to_copy }} + SRC="${SNAPSHOT_DIR}{{ $dest }}" + DEST="/host{{ $dest }}" + if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then + mkdir -p $(dirname "${DEST}") + cp "${SRC}" "${DEST}" + chmod go-rwx "${DEST}" fi - {{end}} + {{- end}} + {{ range $key, $val := .Values.conf }} + SRC="${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}" + DEST="/host/etc/kubernetes/apiserver/{{ $val.file }}" + if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then + mkdir -p $(dirname "${DEST}") + cp "${SRC}" "${DEST}" + chmod go-rwx "${DEST}" + fi + {{- end }} } cleanup() { - - {{range .Values.anchor.files_to_copy}} - rm -f /host{{ .dest }} - {{end}} + {{- range $dest, $source := .Values.const.files_to_copy }} + rm -f "/host{{ $dest }}" + {{- end }} + {{ range $key, $val := .Values.conf }} + rm -f "/host/{{ $val.file }}" + {{- end }} } -while true; do +SNAPSHOT_DIR=$(mktemp -d) + +snapshot_files "${SNAPSHOT_DIR}" + +while true; do if [ -e /tmp/stop ]; then echo Stopping cleanup @@ -43,7 +71,7 @@ while true; do # Compare and replace files on Genesis host if needed # Copy files to other master nodes - compare_copy_files + compare_copy_files "${SNAPSHOT_DIR}" sleep {{ .Values.anchor.period }} done diff --git a/charts/apiserver/templates/configmap-etc.yaml b/charts/apiserver/templates/configmap-etc.yaml index 75a22eac..016290fd 100644 --- a/charts/apiserver/templates/configmap-etc.yaml +++ b/charts/apiserver/templates/configmap-etc.yaml @@ -17,34 +17,19 @@ limitations under the License. {{- if .Values.manifests.configmap_etc }} {{- $envAll := . }} -{{/* This slightly involved merge of AC config files into the anchor - files uses HTK merge, as straighforward appends result in duplicates. */}} -{{- $_ := set .Values "_ac_files_to_copy" list }} -{{- range $key, $val := .Values.conf.admission_controllers }} - {{- $source := printf "/tmp/etc/%s" $key }} - {{- $dest := printf "/etc/kubernetes/apiserver/%s" $key }} - {{- $file_to_copy := dict "source" $source "dest" $dest }} - {{- $ac_files_to_copy := append $.Values._ac_files_to_copy $file_to_copy }} - {{- $_ := set $.Values "_ac_files_to_copy" $ac_files_to_copy }} -{{- end }} -{{ $all_files_to_copy := dict }} -{{ $_ := set $all_files_to_copy "values" (tuple .Values.anchor.files_to_copy .Values._ac_files_to_copy) }} -{{ $_ := $all_files_to_copy | include "helm-toolkit.utils.merge" }} -{{ $_ := set .Values.anchor "files_to_copy" $all_files_to_copy.result }} - --- apiVersion: v1 kind: ConfigMap metadata: name: {{ .Values.service.name }}-etc data: - kubernetes-apiserver.yaml: |+ + kubernetes-apiserver.yaml: | {{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - kubeconfig.yaml: |+ + kubeconfig.yaml: | {{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{/* Dynamically add config files for admission controllers */}} -{{ range $key, $val := .Values.conf.admission_controllers }} - {{ $key }}: |+ -{{ toYaml $val | indent 4 }} -{{ end }} +{{/* Dynamically added config files */}} +{{- range $key, $val := .Values.conf }} + {{ $val.file }}: | +{{ toYaml $val.content | indent 4 }} +{{- end }} {{- end }} diff --git a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl index daf04e13..73f6ccfc 100644 --- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl +++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl @@ -42,30 +42,25 @@ spec: fieldPath: spec.nodeName - name: KUBECONFIG value: /etc/kubernetes/apiserver/kubeconfig.yaml + - name: APISERVER_PORT + value: {{ .Values.network.kubernetes_apiserver.port | quote }} + - name: ETCD_ENDPOINTS + value: {{ .Values.apiserver.etcd.endpoints | quote }} command: - {{- range .Values.command_prefix }} + {{- range .Values.const.command_prefix }} - {{ . }} {{- end }} - - --advertise-address=$(POD_IP) - - --anonymous-auth=false - - --bind-address=0.0.0.0 - - --secure-port={{ .Values.network.kubernetes_apiserver.port }} - - --insecure-port=0 - - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem - - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem - - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem - - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem - - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem - - --etcd-servers={{ .Values.apiserver.etcd.endpoints }} - - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem - - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem - - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem - - --allow-privileged=true - - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub - - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml + {{- range .Values.apiserver.arguments }} + - {{ . }} + {{- end }} + {{- range $key, $val := .Values.conf }} + {{- if hasKey $val "command_options" }} + {{- range $val.command_options }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} ports: - containerPort: {{ .Values.network.kubernetes_apiserver.port }} diff --git a/charts/apiserver/values.yaml b/charts/apiserver/values.yaml index 9c0556bd..b7c5ecf7 100644 --- a/charts/apiserver/values.yaml +++ b/charts/apiserver/values.yaml @@ -14,6 +14,45 @@ release_group: null +# NOTE(mark-burnett): These values are not really configurable -- they live +# here to keep the templates cleaner. +const: + command_prefix: + - /apiserver + - --advertise-address=$(POD_IP) + - --allow-privileged=true + - --anonymous-auth=false + - --bind-address=0.0.0.0 + - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem + - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem + - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem + - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem + - --etcd-servers=$(ETCD_ENDPOINTS) + - --insecure-port=0 + - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem + - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem + - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --secure-port=$(APISERVER_PORT) + - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub + - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem + - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem + + files_to_copy: + # NOTE(mark-burnett): These are (host dest): (container source) pairs + /etc/kubernetes/apiserver/kubeconfig.yaml: /tmp/etc/kubeconfig.yaml + /etc/kubernetes/apiserver/pki/apiserver-key.pem: /keys/apiserver-key.pem + /etc/kubernetes/apiserver/pki/apiserver.pem: /certs/apiserver.pem + /etc/kubernetes/apiserver/pki/cluster-ca.pem: /certs/cluster-ca.pem + /etc/kubernetes/apiserver/pki/etcd-client-ca.pem: /certs/etcd-client-ca.pem + /etc/kubernetes/apiserver/pki/etcd-client-key.pem: /keys/etcd-client-key.pem + /etc/kubernetes/apiserver/pki/etcd-client.pem: /certs/etcd-client.pem + /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem: /certs/kubelet-client-ca.pem + /etc/kubernetes/apiserver/pki/kubelet-client-key.pem: /keys/kubelet-client-key.pem + /etc/kubernetes/apiserver/pki/kubelet-client.pem: /certs/kubelet-client.pem + /etc/kubernetes/apiserver/pki/service-account.pub: /certs/service-account.pub + /etc/kubernetes/manifests/kubernetes-apiserver.yaml: /tmp/etc/kubernetes-apiserver.yaml + images: tags: anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.11 @@ -30,65 +69,58 @@ anchor: kubelet: manifest_path: /etc/kubernetes/manifests period: 15 - files_to_copy: - - source: /certs/apiserver.pem - dest: /etc/kubernetes/apiserver/pki/apiserver.pem - - source: /certs/kubelet-client.pem - dest: /etc/kubernetes/apiserver/pki/kubelet-client.pem - - source: /certs/kubelet-client-ca.pem - dest: /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem - - source: /certs/cluster-ca.pem - dest: /etc/kubernetes/apiserver/pki/cluster-ca.pem - - source: /certs/etcd-client-ca.pem - dest: /etc/kubernetes/apiserver/pki/etcd-client-ca.pem - - source: /certs/etcd-client.pem - dest: /etc/kubernetes/apiserver/pki/etcd-client.pem - - source: /certs/service-account.pub - dest: /etc/kubernetes/apiserver/pki/service-account.pub - - source: /keys/apiserver-key.pem - dest: /etc/kubernetes/apiserver/pki/apiserver-key.pem - - source: /keys/kubelet-client-key.pem - dest: /etc/kubernetes/apiserver/pki/kubelet-client-key.pem - - source: /keys/etcd-client-key.pem - dest: /etc/kubernetes/apiserver/pki/etcd-client-key.pem - - source: /tmp/etc/kubernetes-apiserver.yaml - dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml - - source: /tmp/etc/kubeconfig.yaml - dest: /etc/kubernetes/apiserver/kubeconfig.yaml - # Note: config files for admission controllers are added to this dynamically - -command_prefix: - - /apiserver - - --authorization-mode=Node,RBAC - - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit - - --service-cluster-ip-range=10.96.0.0/16 - - --endpoint-reconciler-type=lease - # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 - - --repair-malformed-updates=false - -apiserver: - host_etc_path: /etc/kubernetes/apiserver - etcd: - endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local conf: - # Admission controllers config files are generated dynamically based on the - # config below, as they are specific to particular ACs that may be - # configured by the operator (or added by k8s in the future). - admission_controllers: - eventconfig.yaml: - kind: Configuration - apiVersion: eventratelimit.admission.k8s.io/v1alpha1 - limits: - - type: Server - qps: 100 - burst: 1000 - acconfig.yaml: - kind: AdmissionConfiguration - apiVersion: apiserver.k8s.io/v1alpha1 - plugins: - - name: EventRateLimit - path: eventconfig.yaml +# Uncomment any of the below to enable the file placement and associated apiserver +# command line options +# +# acconfig: +# file: acconfig.yaml +# command_options: +# - '--admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml' +# - '--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit' +# content: +# kind: AdmissionConfiguration +# apiVersion: apiserver.k8s.io/v1alpha1 +# plugins: +# - name: EventRateLimit +# path: eventconfig.yaml +# eventconfig: +# file: eventconfig.yaml +# command_options: +# - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml' +# content: +# kind: Configuration +# apiVersion: eventratelimit.admission.k8s.io/v1alpha1 +# limits: +# - type: Server +# qps: 1000 +# burst: 10000 +# encryption_provider: +# file: encryption_provider.yaml +# command_option: '' +# content: +# kind: EncryptionConfig +# apiVersion: v1 +# resources: +# - resources: +# - 'secrets' +# providers: +# - identity: {} + +apiserver: + arguments: + - --authorization-mode=Node,RBAC + - --service-cluster-ip-range=10.96.0.0/16 + - --endpoint-reconciler-type=lease + - --feature-gates=PodShareProcessNamespace=true + # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 + - --repair-malformed-updates=false + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction + - --v=3 + etcd: + endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local + host_etc_path: /etc/kubernetes/apiserver network: kubernetes_apiserver: @@ -130,7 +162,6 @@ secrets: cert: null key: null - # typically overriden by environmental # values, but should include all endpoints # required by this chart @@ -170,7 +201,7 @@ pod: upgrades: daemonsets: pod_replacement_strategy: RollingUpdate - kubernetes_apiserver: + kubernetes-apiserver-anchor: enabled: false min_ready_seconds: 0 max_unavailable: 1 diff --git a/examples/basic/Genesis.yaml b/examples/basic/Genesis.yaml index 823ae70a..ddc19168 100644 --- a/examples/basic/Genesis.yaml +++ b/examples/basic/Genesis.yaml @@ -11,15 +11,16 @@ data: hostname: n0 ip: 192.168.77.10 apiserver: - command_prefix: - - /apiserver + arguments: - --authorization-mode=Node,RBAC - - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit - --service-cluster-ip-range=10.96.0.0/16 - --endpoint-reconciler-type=lease - --feature-gates=PodShareProcessNamespace=true # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 - --repair-malformed-updates=false + - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml + - --v=3 armada: target_manifest: cluster-bootstrap labels: @@ -45,4 +46,22 @@ data: - path: /var/lib/anchor/calico-etcd-bootstrap content: "# placeholder for triggering calico etcd bootstrapping" mode: 0644 + # NOTE(mark-burnett): These are referenced by the apiserver arguments above. + - path: /etc/genesis/apiserver/acconfig.yaml + mode: 0444 + content: | + kind: AdmissionConfiguration + apiVersion: apiserver.k8s.io/v1alpha1 + plugins: + - name: EventRateLimit + path: eventconfig.yaml + - path: /etc/genesis/apiserver/eventconfig.yaml + mode: 0444 + content: | + kind: Configuration + apiVersion: eventratelimit.admission.k8s.io/v1alpha1 + limits: + - type: Server + qps: 1000 + burst: 10000 ... diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index f99fdd30..8df50a16 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -719,15 +719,6 @@ data: upgrade: no_hooks: true values: - command_prefix: - - /apiserver - - --authorization-mode=Node,RBAC - - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit - - --service-cluster-ip-range=10.96.0.0/16 - - --endpoint-reconciler-type=lease - - --feature-gates=PodShareProcessNamespace=true - # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11 - - --repair-malformed-updates=false apiserver: etcd: endpoints: https://127.0.0.1:2378 diff --git a/promenade/config.py b/promenade/config.py index 79febba1..6553077d 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -241,7 +241,7 @@ class Configuration: def bootstrap_apiserver_prefix(self): return self.get_path('Genesis:apiserver.command_prefix', - ['/apiserver', '--apiserver-count=2', '--v=5']) + ['/apiserver']) def _matches_filter(document, *, schema, labels, name): diff --git a/promenade/schemas/Genesis.yaml b/promenade/schemas/Genesis.yaml index 12f9b5b5..021f3c38 100644 --- a/promenade/schemas/Genesis.yaml +++ b/promenade/schemas/Genesis.yaml @@ -71,6 +71,10 @@ data: type: array items: type: string + arguments: + type: array + items: + type: string additionalProperties: false files: diff --git a/promenade/templates/include/genesis-apiserver.yaml b/promenade/templates/include/genesis-apiserver.yaml new file mode 100644 index 00000000..4314a61c --- /dev/null +++ b/promenade/templates/include/genesis-apiserver.yaml @@ -0,0 +1,18 @@ + - --advertise-address={{ config['Genesis:ip'] }} + - --allow-privileged=true + - --anonymous-auth=false + - --bind-address=0.0.0.0 + - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem + - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem + - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem + - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem + - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem + - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem + - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub + - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem + - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem + {%- for argument in config.get_path('Genesis:apiserver.arguments', []) %} + - "{{ argument }}" + {%- endfor %} diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml deleted file mode 100644 index c792a8b4..00000000 --- a/promenade/templates/roles/genesis/etc/genesis/apiserver/acconfig.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -kind: AdmissionConfiguration -apiVersion: apiserver.k8s.io/v1alpha1 -plugins: -- name: EventRateLimit - path: eventconfig.yaml \ No newline at end of file diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml deleted file mode 100644 index ae789689..00000000 --- a/promenade/templates/roles/genesis/etc/genesis/apiserver/eventconfig.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -kind: Configuration -apiVersion: eventratelimit.admission.k8s.io/v1alpha1 -limits: -- type: Server - qps: 100 - burst: 1000 \ No newline at end of file diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml index d122d570..e9051aa4 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml @@ -11,146 +11,130 @@ spec: dnsPolicy: Default hostNetwork: true containers: - - env: - - name: TILLER_NAMESPACE - value: kube-system - image: {{ config['Genesis:images.helm.tiller'] }} - command: - - /tiller - - -logtostderr - - -v - - "99" - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /liveness - port: 44135 - scheme: HTTP - initialDelaySeconds: 1 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: tiller - ports: - - containerPort: 44134 + - env: + - name: TILLER_NAMESPACE + value: kube-system + image: {{ config['Genesis:images.helm.tiller'] }} + command: + - /tiller + - -logtostderr + - -v + - "99" + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 44135 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 name: tiller - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readiness - port: 44135 - scheme: HTTP - initialDelaySeconds: 1 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: {} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - - name: armada - image: {{ config['Genesis:images.armada'] }} - securityContext: - runAsUser: 0 - command: - - /bin/bash - - -c - - |- - set -x + ports: + - containerPort: 44134 + name: tiller + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 44135 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + - name: armada + image: {{ config['Genesis:images.armada'] }} + securityContext: + runAsUser: 0 + command: + - /bin/bash + - -c + - |- + set -x - while true; do - sleep 10 - if armada \ - apply \ - --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \ - --tiller-host 127.0.0.1 \ - /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then - break - fi - done + while true; do + sleep 10 + if armada \ + apply \ + --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \ + --tiller-host 127.0.0.1 \ + /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then + break + fi + done + touch /ipc/armada-done + sleep 10000 + env: + - name: ARMADA_LOGFILE + value: /tmp/log/bootstrap-armada.log + {%- if config['KubernetesNetwork:proxy.url'] is defined %} + - name: HTTP_PROXY + value: {{ config['KubernetesNetwork:proxy.url'] }} + - name: HTTPS_PROXY + value: {{ config['KubernetesNetwork:proxy.url'] }} + - name: NO_PROXY + value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }} + - name: http_proxy + value: {{ config['KubernetesNetwork:proxy.url'] }} + - name: https_proxy + value: {{ config['KubernetesNetwork:proxy.url'] }} + - name: no_proxy + value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }} + {%- endif %} + volumeMounts: + - name: assets + mountPath: /etc/genesis/armada/assets + - name: auth + mountPath: /root/.kube + - name: ipc + mountPath: /ipc + - name: log + mountPath: /tmp/log + - name: monitor + image: {{ config['HostSystem:images.kubernetes.kubectl'] }} + command: + - /bin/sh + - -c + - |- + set -x - touch /ipc/armada-done - sleep 10000 - env: - - name: ARMADA_LOGFILE - value: /tmp/log/bootstrap-armada.log -{%- if config['KubernetesNetwork:proxy.url'] is defined %} - - name: HTTP_PROXY - value: {{ config['KubernetesNetwork:proxy.url'] }} - - name: HTTPS_PROXY - value: {{ config['KubernetesNetwork:proxy.url'] }} - - name: NO_PROXY - value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }} - - name: http_proxy - value: {{ config['KubernetesNetwork:proxy.url'] }} - - name: https_proxy - value: {{ config['KubernetesNetwork:proxy.url'] }} - - name: no_proxy - value: {{ config.get(kind='KubernetesNetwork') | fill_no_proxy }} -{%- endif %} - volumeMounts: - - name: assets - mountPath: /etc/genesis/armada/assets - - name: auth - mountPath: /root/.kube - - name: ipc - mountPath: /ipc - - name: log - mountPath: /tmp/log - - name: monitor - image: {{ config['HostSystem:images.kubernetes.kubectl'] }} - command: - - /bin/sh - - -c - - |- - set -x + while ! [ -e /ipc/armada-done ]; do + sleep 5 + done - while ! [ -e /ipc/armada-done ]; do - sleep 5 - done - - rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml - sleep 10000 - volumeMounts: - - name: ipc - mountPath: /ipc - - name: manifest - mountPath: /etc/kubernetes/manifests - - name: kubectl-apiserver - image: {{ config['Genesis:images.kubernetes.apiserver'] }} - command: - {%- for argument in config.bootstrap_apiserver_prefix() %} - - "{{ argument }}" - {%- endfor %} - - --advertise-address={{ config['Genesis:ip'] }} - - --anonymous-auth=false - - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem - - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem - - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem - - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem - - --insecure-port=8080 - - --secure-port=6444 - - --bind-address=0.0.0.0 - - --allow-privileged=true - - --etcd-servers=https://localhost:12379 - - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem - - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem - - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub - - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml - - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem - - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem - env: - - name: KUBECONFIG - value: /etc/kubernetes/admin/config - volumeMounts: - - name: auth - mountPath: /etc/kubernetes/admin - - name: config - mountPath: /etc/kubernetes/apiserver - readOnly: true + rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml + sleep 10000 + volumeMounts: + - name: ipc + mountPath: /ipc + - name: manifest + mountPath: /etc/kubernetes/manifests + - name: kubectl-apiserver + image: {{ config['Genesis:images.kubernetes.apiserver'] }} + command: + {%- for argument in config.bootstrap_apiserver_prefix() %} + - "{{ argument }}" + {%- endfor %} +{% include "genesis-apiserver.yaml" with context %} + - --etcd-servers=https://localhost:12379 + - --insecure-port=8080 + - --secure-port=6444 + env: + - name: KUBECONFIG + value: /etc/kubernetes/admin/config + volumeMounts: + - name: auth + mountPath: /etc/kubernetes/admin + - name: config + mountPath: /etc/kubernetes/apiserver + readOnly: true volumes: - name: assets hostPath: diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml index 606f0f3e..4113327b 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml @@ -19,25 +19,10 @@ spec: {%- for argument in config.bootstrap_apiserver_prefix() %} - "{{ argument }}" {%- endfor %} - - --advertise-address={{ config['Genesis:ip'] }} - - --anonymous-auth=false - - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem - - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem - - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem - - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem - - --insecure-port=0 - - --bind-address=0.0.0.0 - - --secure-port=6443 - - --allow-privileged=true +{% include "genesis-apiserver.yaml" with context %} - --etcd-servers=https://localhost:2379 - - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem - - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem - - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub - - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml - - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem - - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem + - --insecure-port=0 + - --secure-port=6443 volumeMounts: - name: config mountPath: /etc/kubernetes/apiserver