From 51df4ce0788ca2c9892d6e10280dfbbd7600129c Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Fri, 1 Dec 2017 11:06:09 -0600 Subject: [PATCH] Avoid insecure apiserver port for tiller This allows us to replace the apiserver process during genesis with the chart-managed version that is likely to only listen on a secure port. * Bundle armada + tiller + insecure apiserver into a static pod * Report aramda logs via host filesystem NOTE: This is using an additional apiserver sidecar rather than a `kubectl proxy` sidecar with a serviceaccount, because it's running as a static pod. Change-Id: I39c638020c0ad36db8d3b10c4ecb959a6642ad0e --- entrypoint.sh | 1 + promenade/templates/include/utils.sh | 6 +- .../roles/common/etc/coredns/zones/promenade | 2 + .../manifests/bootstrap-armada.yaml | 132 ++++++++++++++++++ .../manifests/kubernetes-apiserver.yaml | 4 +- .../etc/kubernetes/manifests/tiller.yaml | 55 -------- promenade/templates/scripts/genesis.sh | 27 +++- 7 files changed, 164 insertions(+), 63 deletions(-) create mode 100644 promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml delete mode 100644 promenade/templates/roles/genesis/etc/kubernetes/manifests/tiller.yaml diff --git a/entrypoint.sh b/entrypoint.sh index 20e44c8a..82402ad8 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -6,6 +6,7 @@ PORT=${PORT:-9000} if [ "$1" = 'server' ]; then exec uwsgi \ --http :${PORT} \ + -z 300 \ --paste config:/etc/promenade/api-paste.ini \ --enable-threads -L \ --workers 4 diff --git a/promenade/templates/include/utils.sh b/promenade/templates/include/utils.sh index d6523d08..cb79e81f 100644 --- a/promenade/templates/include/utils.sh +++ b/promenade/templates/include/utils.sh @@ -195,19 +195,19 @@ function wait_for_pod_termination { end=$(($(date +%s) + $SEC)) while true; do - POD_PHASE=$(kubectl --request-timeout 10s --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME) + POD_PHASE=$(kubectl --request-timeout 10s --namespace $NAMESPACE get -a -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME) if [ "x$POD_PHASE" = "xSucceeded" ]; then log Pod $POD_NAME succeeded. break elif [ "x$POD_PHASE" = "xFailed" ]; then log Pod $POD_NAME failed. - kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 + kubectl --request-timeout 10s --namespace $NAMESPACE get -a -o yaml pod $POD_NAME 1>&2 fail else now=$(date +%s) if [ $now -gt $end ]; then log Pod did not terminate before timeout. - kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 + kubectl --request-timeout 10s --namespace $NAMESPACE get -a -o yaml pod $POD_NAME 1>&2 fail fi sleep 1 diff --git a/promenade/templates/roles/common/etc/coredns/zones/promenade b/promenade/templates/roles/common/etc/coredns/zones/promenade index 316e9bed..4b32f39a 100644 --- a/promenade/templates/roles/common/etc/coredns/zones/promenade +++ b/promenade/templates/roles/common/etc/coredns/zones/promenade @@ -7,3 +7,5 @@ apiserver.kubernetes IN A {{ config['KubernetesNode:join_ip'] }} {%- else %} apiserver.kubernetes IN A 127.0.0.1 {%- endif %} + +etcd.kubernetes IN A 127.0.0.1 diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml new file mode 100644 index 00000000..fefc0702 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml @@ -0,0 +1,132 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: bootstrap-armada + namespace: kube-system + labels: + app: promenade + component: genesis-tiller +spec: + dnsPolicy: Default + hostNetwork: true + containers: + - env: + - name: TILLER_NAMESPACE + value: kube-system + image: {{ config['Genesis:images.helm.tiller'] }} + command: + - /tiller + - -logtostderr + - -v + - "99" + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 44135 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: tiller + ports: + - containerPort: 44134 + name: tiller + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 44135 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + - name: armada + image: {{ config['Genesis:images.armada'] }} + command: + - /bin/bash + - -c + - |- + set -x + + while true; do + sleep 10 + if armada --debug apply --tiller-host 127.0.0.1 /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then + break + fi + done + + touch /ipc/armada-done + sleep 10000 + env: + - name: ARMADA_LOGFILE + value: /tmp/log/bootstrap-armada.log + volumeMounts: + - name: assets + mountPath: /etc/genesis/armada/assets + - name: auth + mountPath: /armada/.kube + - name: ipc + mountPath: /ipc + - name: log + mountPath: /tmp/log + - name: monitor + image: {{ config['HostSystem:images.kubernetes.kubectl'] }} + command: + - /bin/sh + - -c + - |- + set -x + + while ! [ -e /ipc/armada-done ]; do + sleep 5 + done + + rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml + sleep 10000 + volumeMounts: + - name: ipc + mountPath: /ipc + - name: manifest + mountPath: /etc/kubernetes/manifests + - name: kubectl-proxy + image: {{ config['HostSystem:images.kubernetes.kubectl'] }} + command: + - kubectl + - proxy + - --port=8080 + env: + - name: KUBECONFIG + value: /etc/kubernetes/admin/config + volumeMounts: + - name: auth + mountPath: /etc/kubernetes/admin + volumes: + - name: assets + hostPath: + path: /etc/genesis/armada/assets + - name: auth + hostPath: + path: /etc/genesis/armada/auth + - name: manifest + hostPath: + path: /etc/kubernetes/manifests + - name: ipc + emptyDir: {} + - name: log + hostPath: + path: /var/log/armada + + + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml index aee3d4d7..37218b18 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml @@ -27,9 +27,7 @@ spec: # Hard coding 3 is a pretty safe move for now. This can be exposed # with additional configuration later. - --apiserver-count=3 - # XXX Temporarily enabled for tiller - - --insecure-port=8080 - - --insecure-bind-address=127.0.0.1 + - --insecure-port=0 - --bind-address=0.0.0.0 - --secure-port=6443 - --runtime-config=batch/v2alpha1=true diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/tiller.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/tiller.yaml deleted file mode 100644 index a694e993..00000000 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/tiller.yaml +++ /dev/null @@ -1,55 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: tiller-deploy - namespace: kube-system - labels: - app: promenade - component: genesis-tiller -spec: - dnsPolicy: Default - hostNetwork: true - containers: - - env: - - name: TILLER_NAMESPACE - value: kube-system - image: {{ config['Genesis:images.helm.tiller'] }} - command: - - /tiller - - -logtostderr - - -v - - "99" - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /liveness - port: 44135 - scheme: HTTP - initialDelaySeconds: 1 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: tiller - ports: - - containerPort: 44134 - name: tiller - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readiness - port: 44135 - scheme: HTTP - initialDelaySeconds: 1 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: {} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - restartPolicy: Always - schedulerName: default-scheduler - securityContext: {} - terminationGracePeriodSeconds: 30 diff --git a/promenade/templates/scripts/genesis.sh b/promenade/templates/scripts/genesis.sh index f0cf6df7..795cfe92 100644 --- a/promenade/templates/scripts/genesis.sh +++ b/promenade/templates/scripts/genesis.sh @@ -2,12 +2,17 @@ {% include "up.sh" with context %} +mkdir -p /var/log/armada +touch /var/log/armada/bootstrap-armada.log +chmod 777 /var/log/armada/bootstrap-armada.log + set +x log log === Waiting for Kubernetes API availablity === set -x wait_for_kubernetes_api 3600 + {%- if config['Genesis:labels.dynamic'] is defined %} set +x log @@ -21,12 +26,30 @@ log log === Deploying bootstrap manifest via Armada === set -x +while [[ ! -e /var/log/armada/bootstrap-armada.log ]]; do + sleep 5 +done +tail -f /var/log/armada/bootstrap-armada.log & + +set +x +end=$(($(date +%s) + 3600)) while true; do - sleep 10 - if armada apply --debug /etc/genesis/armada/assets/manifest.yaml ; then + if [[ -e /etc/kubernetes/manifests/bootstrap-armada.yaml ]]; then + now=$(date +%s) + if [ $now -gt $end ]; then + log Armada static pod manifest still in place after expected duration + fail + fi + sleep 5 + else + log Armada static pod manifest removed break fi done +set -x + +# Terminate background job (tear down exit trap?) +kill %1 set +x log