Avoid insecure apiserver port for tiller
This allows us to replace the apiserver process during genesis with the chart-managed version that is likely to only listen on a secure port. * Bundle armada + tiller + insecure apiserver into a static pod * Report aramda logs via host filesystem NOTE: This is using an additional apiserver sidecar rather than a `kubectl proxy` sidecar with a serviceaccount, because it's running as a static pod. Change-Id: I39c638020c0ad36db8d3b10c4ecb959a6642ad0e
This commit is contained in:
Mark Burnett
parent
af35ac2f2b
commit
51df4ce078
|
|
@ -6,6 +6,7 @@ PORT=${PORT:-9000}
|
|||
if [ "$1" = 'server' ]; then
|
||||
exec uwsgi \
|
||||
--http :${PORT} \
|
||||
-z 300 \
|
||||
--paste config:/etc/promenade/api-paste.ini \
|
||||
--enable-threads -L \
|
||||
--workers 4
|
||||
|
|
|
|||
|
|
@ -195,19 +195,19 @@ function wait_for_pod_termination {
|
|||
|
||||
end=$(($(date +%s) + $SEC))
|
||||
while true; do
|
||||
POD_PHASE=$(kubectl --request-timeout 10s --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME)
|
||||
POD_PHASE=$(kubectl --request-timeout 10s --namespace $NAMESPACE get -a -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME)
|
||||
if [ "x$POD_PHASE" = "xSucceeded" ]; then
|
||||
log Pod $POD_NAME succeeded.
|
||||
break
|
||||
elif [ "x$POD_PHASE" = "xFailed" ]; then
|
||||
log Pod $POD_NAME failed.
|
||||
kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
|
||||
kubectl --request-timeout 10s --namespace $NAMESPACE get -a -o yaml pod $POD_NAME 1>&2
|
||||
fail
|
||||
else
|
||||
now=$(date +%s)
|
||||
if [ $now -gt $end ]; then
|
||||
log Pod did not terminate before timeout.
|
||||
kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
|
||||
kubectl --request-timeout 10s --namespace $NAMESPACE get -a -o yaml pod $POD_NAME 1>&2
|
||||
fail
|
||||
fi
|
||||
sleep 1
|
||||
|
|
|
|||
|
|
@ -7,3 +7,5 @@ apiserver.kubernetes IN A {{ config['KubernetesNode:join_ip'] }}
|
|||
{%- else %}
|
||||
apiserver.kubernetes IN A 127.0.0.1
|
||||
{%- endif %}
|
||||
|
||||
etcd.kubernetes IN A 127.0.0.1
|
||||
|
|
|
|||
|
|
@ -0,0 +1,132 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: bootstrap-armada
|
||||
namespace: kube-system
|
||||
labels:
|
||||
app: promenade
|
||||
component: genesis-tiller
|
||||
spec:
|
||||
dnsPolicy: Default
|
||||
hostNetwork: true
|
||||
containers:
|
||||
- env:
|
||||
- name: TILLER_NAMESPACE
|
||||
value: kube-system
|
||||
image: {{ config['Genesis:images.helm.tiller'] }}
|
||||
command:
|
||||
- /tiller
|
||||
- -logtostderr
|
||||
- -v
|
||||
- "99"
|
||||
imagePullPolicy: IfNotPresent
|
||||
livenessProbe:
|
||||
failureThreshold: 3
|
||||
httpGet:
|
||||
path: /liveness
|
||||
port: 44135
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 1
|
||||
periodSeconds: 10
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 1
|
||||
name: tiller
|
||||
ports:
|
||||
- containerPort: 44134
|
||||
name: tiller
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 3
|
||||
httpGet:
|
||||
path: /readiness
|
||||
port: 44135
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 1
|
||||
periodSeconds: 10
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 1
|
||||
resources: {}
|
||||
terminationMessagePath: /dev/termination-log
|
||||
terminationMessagePolicy: File
|
||||
- name: armada
|
||||
image: {{ config['Genesis:images.armada'] }}
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |-
|
||||
set -x
|
||||
|
||||
while true; do
|
||||
sleep 10
|
||||
if armada --debug apply --tiller-host 127.0.0.1 /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
touch /ipc/armada-done
|
||||
sleep 10000
|
||||
env:
|
||||
- name: ARMADA_LOGFILE
|
||||
value: /tmp/log/bootstrap-armada.log
|
||||
volumeMounts:
|
||||
- name: assets
|
||||
mountPath: /etc/genesis/armada/assets
|
||||
- name: auth
|
||||
mountPath: /armada/.kube
|
||||
- name: ipc
|
||||
mountPath: /ipc
|
||||
- name: log
|
||||
mountPath: /tmp/log
|
||||
- name: monitor
|
||||
image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |-
|
||||
set -x
|
||||
|
||||
while ! [ -e /ipc/armada-done ]; do
|
||||
sleep 5
|
||||
done
|
||||
|
||||
rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml
|
||||
sleep 10000
|
||||
volumeMounts:
|
||||
- name: ipc
|
||||
mountPath: /ipc
|
||||
- name: manifest
|
||||
mountPath: /etc/kubernetes/manifests
|
||||
- name: kubectl-proxy
|
||||
image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
|
||||
command:
|
||||
- kubectl
|
||||
- proxy
|
||||
- --port=8080
|
||||
env:
|
||||
- name: KUBECONFIG
|
||||
value: /etc/kubernetes/admin/config
|
||||
volumeMounts:
|
||||
- name: auth
|
||||
mountPath: /etc/kubernetes/admin
|
||||
volumes:
|
||||
- name: assets
|
||||
hostPath:
|
||||
path: /etc/genesis/armada/assets
|
||||
- name: auth
|
||||
hostPath:
|
||||
path: /etc/genesis/armada/auth
|
||||
- name: manifest
|
||||
hostPath:
|
||||
path: /etc/kubernetes/manifests
|
||||
- name: ipc
|
||||
emptyDir: {}
|
||||
- name: log
|
||||
hostPath:
|
||||
path: /var/log/armada
|
||||
|
||||
|
||||
restartPolicy: Always
|
||||
schedulerName: default-scheduler
|
||||
securityContext: {}
|
||||
terminationGracePeriodSeconds: 30
|
||||
|
|
@ -27,9 +27,7 @@ spec:
|
|||
# Hard coding 3 is a pretty safe move for now. This can be exposed
|
||||
# with additional configuration later.
|
||||
- --apiserver-count=3
|
||||
# XXX Temporarily enabled for tiller
|
||||
- --insecure-port=8080
|
||||
- --insecure-bind-address=127.0.0.1
|
||||
- --insecure-port=0
|
||||
- --bind-address=0.0.0.0
|
||||
- --secure-port=6443
|
||||
- --runtime-config=batch/v2alpha1=true
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: tiller-deploy
|
||||
namespace: kube-system
|
||||
labels:
|
||||
app: promenade
|
||||
component: genesis-tiller
|
||||
spec:
|
||||
dnsPolicy: Default
|
||||
hostNetwork: true
|
||||
containers:
|
||||
- env:
|
||||
- name: TILLER_NAMESPACE
|
||||
value: kube-system
|
||||
image: {{ config['Genesis:images.helm.tiller'] }}
|
||||
command:
|
||||
- /tiller
|
||||
- -logtostderr
|
||||
- -v
|
||||
- "99"
|
||||
imagePullPolicy: IfNotPresent
|
||||
livenessProbe:
|
||||
failureThreshold: 3
|
||||
httpGet:
|
||||
path: /liveness
|
||||
port: 44135
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 1
|
||||
periodSeconds: 10
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 1
|
||||
name: tiller
|
||||
ports:
|
||||
- containerPort: 44134
|
||||
name: tiller
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 3
|
||||
httpGet:
|
||||
path: /readiness
|
||||
port: 44135
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 1
|
||||
periodSeconds: 10
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 1
|
||||
resources: {}
|
||||
terminationMessagePath: /dev/termination-log
|
||||
terminationMessagePolicy: File
|
||||
restartPolicy: Always
|
||||
schedulerName: default-scheduler
|
||||
securityContext: {}
|
||||
terminationGracePeriodSeconds: 30
|
||||
|
|
@ -2,12 +2,17 @@
|
|||
|
||||
{% include "up.sh" with context %}
|
||||
|
||||
mkdir -p /var/log/armada
|
||||
touch /var/log/armada/bootstrap-armada.log
|
||||
chmod 777 /var/log/armada/bootstrap-armada.log
|
||||
|
||||
set +x
|
||||
log
|
||||
log === Waiting for Kubernetes API availablity ===
|
||||
set -x
|
||||
wait_for_kubernetes_api 3600
|
||||
|
||||
|
||||
{%- if config['Genesis:labels.dynamic'] is defined %}
|
||||
set +x
|
||||
log
|
||||
|
|
@ -21,12 +26,30 @@ log
|
|||
log === Deploying bootstrap manifest via Armada ===
|
||||
set -x
|
||||
|
||||
while [[ ! -e /var/log/armada/bootstrap-armada.log ]]; do
|
||||
sleep 5
|
||||
done
|
||||
tail -f /var/log/armada/bootstrap-armada.log &
|
||||
|
||||
set +x
|
||||
end=$(($(date +%s) + 3600))
|
||||
while true; do
|
||||
sleep 10
|
||||
if armada apply --debug /etc/genesis/armada/assets/manifest.yaml ; then
|
||||
if [[ -e /etc/kubernetes/manifests/bootstrap-armada.yaml ]]; then
|
||||
now=$(date +%s)
|
||||
if [ $now -gt $end ]; then
|
||||
log Armada static pod manifest still in place after expected duration
|
||||
fail
|
||||
fi
|
||||
sleep 5
|
||||
else
|
||||
log Armada static pod manifest removed
|
||||
break
|
||||
fi
|
||||
done
|
||||
set -x
|
||||
|
||||
# Terminate background job (tear down exit trap?)
|
||||
kill %1
|
||||
|
||||
set +x
|
||||
log
|
||||
|
|
|
|||
Loading…
Reference in New Issue