Avoid insecure apiserver port for tiller
This allows us to replace the apiserver process during genesis with the chart-managed version that is likely to only listen on a secure port. * Bundle armada + tiller + insecure apiserver into a static pod * Report aramda logs via host filesystem NOTE: This is using an additional apiserver sidecar rather than a `kubectl proxy` sidecar with a serviceaccount, because it's running as a static pod. Change-Id: I39c638020c0ad36db8d3b10c4ecb959a6642ad0e
This commit is contained in:
parent
af35ac2f2b
commit
51df4ce078
|
@ -6,6 +6,7 @@ PORT=${PORT:-9000}
|
||||||
if [ "$1" = 'server' ]; then
|
if [ "$1" = 'server' ]; then
|
||||||
exec uwsgi \
|
exec uwsgi \
|
||||||
--http :${PORT} \
|
--http :${PORT} \
|
||||||
|
-z 300 \
|
||||||
--paste config:/etc/promenade/api-paste.ini \
|
--paste config:/etc/promenade/api-paste.ini \
|
||||||
--enable-threads -L \
|
--enable-threads -L \
|
||||||
--workers 4
|
--workers 4
|
||||||
|
|
|
@ -195,19 +195,19 @@ function wait_for_pod_termination {
|
||||||
|
|
||||||
end=$(($(date +%s) + $SEC))
|
end=$(($(date +%s) + $SEC))
|
||||||
while true; do
|
while true; do
|
||||||
POD_PHASE=$(kubectl --request-timeout 10s --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME)
|
POD_PHASE=$(kubectl --request-timeout 10s --namespace $NAMESPACE get -a -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME)
|
||||||
if [ "x$POD_PHASE" = "xSucceeded" ]; then
|
if [ "x$POD_PHASE" = "xSucceeded" ]; then
|
||||||
log Pod $POD_NAME succeeded.
|
log Pod $POD_NAME succeeded.
|
||||||
break
|
break
|
||||||
elif [ "x$POD_PHASE" = "xFailed" ]; then
|
elif [ "x$POD_PHASE" = "xFailed" ]; then
|
||||||
log Pod $POD_NAME failed.
|
log Pod $POD_NAME failed.
|
||||||
kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
|
kubectl --request-timeout 10s --namespace $NAMESPACE get -a -o yaml pod $POD_NAME 1>&2
|
||||||
fail
|
fail
|
||||||
else
|
else
|
||||||
now=$(date +%s)
|
now=$(date +%s)
|
||||||
if [ $now -gt $end ]; then
|
if [ $now -gt $end ]; then
|
||||||
log Pod did not terminate before timeout.
|
log Pod did not terminate before timeout.
|
||||||
kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
|
kubectl --request-timeout 10s --namespace $NAMESPACE get -a -o yaml pod $POD_NAME 1>&2
|
||||||
fail
|
fail
|
||||||
fi
|
fi
|
||||||
sleep 1
|
sleep 1
|
||||||
|
|
|
@ -7,3 +7,5 @@ apiserver.kubernetes IN A {{ config['KubernetesNode:join_ip'] }}
|
||||||
{%- else %}
|
{%- else %}
|
||||||
apiserver.kubernetes IN A 127.0.0.1
|
apiserver.kubernetes IN A 127.0.0.1
|
||||||
{%- endif %}
|
{%- endif %}
|
||||||
|
|
||||||
|
etcd.kubernetes IN A 127.0.0.1
|
||||||
|
|
|
@ -0,0 +1,132 @@
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: bootstrap-armada
|
||||||
|
namespace: kube-system
|
||||||
|
labels:
|
||||||
|
app: promenade
|
||||||
|
component: genesis-tiller
|
||||||
|
spec:
|
||||||
|
dnsPolicy: Default
|
||||||
|
hostNetwork: true
|
||||||
|
containers:
|
||||||
|
- env:
|
||||||
|
- name: TILLER_NAMESPACE
|
||||||
|
value: kube-system
|
||||||
|
image: {{ config['Genesis:images.helm.tiller'] }}
|
||||||
|
command:
|
||||||
|
- /tiller
|
||||||
|
- -logtostderr
|
||||||
|
- -v
|
||||||
|
- "99"
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
livenessProbe:
|
||||||
|
failureThreshold: 3
|
||||||
|
httpGet:
|
||||||
|
path: /liveness
|
||||||
|
port: 44135
|
||||||
|
scheme: HTTP
|
||||||
|
initialDelaySeconds: 1
|
||||||
|
periodSeconds: 10
|
||||||
|
successThreshold: 1
|
||||||
|
timeoutSeconds: 1
|
||||||
|
name: tiller
|
||||||
|
ports:
|
||||||
|
- containerPort: 44134
|
||||||
|
name: tiller
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 3
|
||||||
|
httpGet:
|
||||||
|
path: /readiness
|
||||||
|
port: 44135
|
||||||
|
scheme: HTTP
|
||||||
|
initialDelaySeconds: 1
|
||||||
|
periodSeconds: 10
|
||||||
|
successThreshold: 1
|
||||||
|
timeoutSeconds: 1
|
||||||
|
resources: {}
|
||||||
|
terminationMessagePath: /dev/termination-log
|
||||||
|
terminationMessagePolicy: File
|
||||||
|
- name: armada
|
||||||
|
image: {{ config['Genesis:images.armada'] }}
|
||||||
|
command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- |-
|
||||||
|
set -x
|
||||||
|
|
||||||
|
while true; do
|
||||||
|
sleep 10
|
||||||
|
if armada --debug apply --tiller-host 127.0.0.1 /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
touch /ipc/armada-done
|
||||||
|
sleep 10000
|
||||||
|
env:
|
||||||
|
- name: ARMADA_LOGFILE
|
||||||
|
value: /tmp/log/bootstrap-armada.log
|
||||||
|
volumeMounts:
|
||||||
|
- name: assets
|
||||||
|
mountPath: /etc/genesis/armada/assets
|
||||||
|
- name: auth
|
||||||
|
mountPath: /armada/.kube
|
||||||
|
- name: ipc
|
||||||
|
mountPath: /ipc
|
||||||
|
- name: log
|
||||||
|
mountPath: /tmp/log
|
||||||
|
- name: monitor
|
||||||
|
image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -c
|
||||||
|
- |-
|
||||||
|
set -x
|
||||||
|
|
||||||
|
while ! [ -e /ipc/armada-done ]; do
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
|
||||||
|
rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml
|
||||||
|
sleep 10000
|
||||||
|
volumeMounts:
|
||||||
|
- name: ipc
|
||||||
|
mountPath: /ipc
|
||||||
|
- name: manifest
|
||||||
|
mountPath: /etc/kubernetes/manifests
|
||||||
|
- name: kubectl-proxy
|
||||||
|
image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
|
||||||
|
command:
|
||||||
|
- kubectl
|
||||||
|
- proxy
|
||||||
|
- --port=8080
|
||||||
|
env:
|
||||||
|
- name: KUBECONFIG
|
||||||
|
value: /etc/kubernetes/admin/config
|
||||||
|
volumeMounts:
|
||||||
|
- name: auth
|
||||||
|
mountPath: /etc/kubernetes/admin
|
||||||
|
volumes:
|
||||||
|
- name: assets
|
||||||
|
hostPath:
|
||||||
|
path: /etc/genesis/armada/assets
|
||||||
|
- name: auth
|
||||||
|
hostPath:
|
||||||
|
path: /etc/genesis/armada/auth
|
||||||
|
- name: manifest
|
||||||
|
hostPath:
|
||||||
|
path: /etc/kubernetes/manifests
|
||||||
|
- name: ipc
|
||||||
|
emptyDir: {}
|
||||||
|
- name: log
|
||||||
|
hostPath:
|
||||||
|
path: /var/log/armada
|
||||||
|
|
||||||
|
|
||||||
|
restartPolicy: Always
|
||||||
|
schedulerName: default-scheduler
|
||||||
|
securityContext: {}
|
||||||
|
terminationGracePeriodSeconds: 30
|
|
@ -27,9 +27,7 @@ spec:
|
||||||
# Hard coding 3 is a pretty safe move for now. This can be exposed
|
# Hard coding 3 is a pretty safe move for now. This can be exposed
|
||||||
# with additional configuration later.
|
# with additional configuration later.
|
||||||
- --apiserver-count=3
|
- --apiserver-count=3
|
||||||
# XXX Temporarily enabled for tiller
|
- --insecure-port=0
|
||||||
- --insecure-port=8080
|
|
||||||
- --insecure-bind-address=127.0.0.1
|
|
||||||
- --bind-address=0.0.0.0
|
- --bind-address=0.0.0.0
|
||||||
- --secure-port=6443
|
- --secure-port=6443
|
||||||
- --runtime-config=batch/v2alpha1=true
|
- --runtime-config=batch/v2alpha1=true
|
||||||
|
|
|
@ -1,55 +0,0 @@
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
name: tiller-deploy
|
|
||||||
namespace: kube-system
|
|
||||||
labels:
|
|
||||||
app: promenade
|
|
||||||
component: genesis-tiller
|
|
||||||
spec:
|
|
||||||
dnsPolicy: Default
|
|
||||||
hostNetwork: true
|
|
||||||
containers:
|
|
||||||
- env:
|
|
||||||
- name: TILLER_NAMESPACE
|
|
||||||
value: kube-system
|
|
||||||
image: {{ config['Genesis:images.helm.tiller'] }}
|
|
||||||
command:
|
|
||||||
- /tiller
|
|
||||||
- -logtostderr
|
|
||||||
- -v
|
|
||||||
- "99"
|
|
||||||
imagePullPolicy: IfNotPresent
|
|
||||||
livenessProbe:
|
|
||||||
failureThreshold: 3
|
|
||||||
httpGet:
|
|
||||||
path: /liveness
|
|
||||||
port: 44135
|
|
||||||
scheme: HTTP
|
|
||||||
initialDelaySeconds: 1
|
|
||||||
periodSeconds: 10
|
|
||||||
successThreshold: 1
|
|
||||||
timeoutSeconds: 1
|
|
||||||
name: tiller
|
|
||||||
ports:
|
|
||||||
- containerPort: 44134
|
|
||||||
name: tiller
|
|
||||||
protocol: TCP
|
|
||||||
readinessProbe:
|
|
||||||
failureThreshold: 3
|
|
||||||
httpGet:
|
|
||||||
path: /readiness
|
|
||||||
port: 44135
|
|
||||||
scheme: HTTP
|
|
||||||
initialDelaySeconds: 1
|
|
||||||
periodSeconds: 10
|
|
||||||
successThreshold: 1
|
|
||||||
timeoutSeconds: 1
|
|
||||||
resources: {}
|
|
||||||
terminationMessagePath: /dev/termination-log
|
|
||||||
terminationMessagePolicy: File
|
|
||||||
restartPolicy: Always
|
|
||||||
schedulerName: default-scheduler
|
|
||||||
securityContext: {}
|
|
||||||
terminationGracePeriodSeconds: 30
|
|
|
@ -2,12 +2,17 @@
|
||||||
|
|
||||||
{% include "up.sh" with context %}
|
{% include "up.sh" with context %}
|
||||||
|
|
||||||
|
mkdir -p /var/log/armada
|
||||||
|
touch /var/log/armada/bootstrap-armada.log
|
||||||
|
chmod 777 /var/log/armada/bootstrap-armada.log
|
||||||
|
|
||||||
set +x
|
set +x
|
||||||
log
|
log
|
||||||
log === Waiting for Kubernetes API availablity ===
|
log === Waiting for Kubernetes API availablity ===
|
||||||
set -x
|
set -x
|
||||||
wait_for_kubernetes_api 3600
|
wait_for_kubernetes_api 3600
|
||||||
|
|
||||||
|
|
||||||
{%- if config['Genesis:labels.dynamic'] is defined %}
|
{%- if config['Genesis:labels.dynamic'] is defined %}
|
||||||
set +x
|
set +x
|
||||||
log
|
log
|
||||||
|
@ -21,12 +26,30 @@ log
|
||||||
log === Deploying bootstrap manifest via Armada ===
|
log === Deploying bootstrap manifest via Armada ===
|
||||||
set -x
|
set -x
|
||||||
|
|
||||||
|
while [[ ! -e /var/log/armada/bootstrap-armada.log ]]; do
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
tail -f /var/log/armada/bootstrap-armada.log &
|
||||||
|
|
||||||
|
set +x
|
||||||
|
end=$(($(date +%s) + 3600))
|
||||||
while true; do
|
while true; do
|
||||||
sleep 10
|
if [[ -e /etc/kubernetes/manifests/bootstrap-armada.yaml ]]; then
|
||||||
if armada apply --debug /etc/genesis/armada/assets/manifest.yaml ; then
|
now=$(date +%s)
|
||||||
|
if [ $now -gt $end ]; then
|
||||||
|
log Armada static pod manifest still in place after expected duration
|
||||||
|
fail
|
||||||
|
fi
|
||||||
|
sleep 5
|
||||||
|
else
|
||||||
|
log Armada static pod manifest removed
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
set -x
|
||||||
|
|
||||||
|
# Terminate background job (tear down exit trap?)
|
||||||
|
kill %1
|
||||||
|
|
||||||
set +x
|
set +x
|
||||||
log
|
log
|
||||||
|
|
Loading…
Reference in New Issue