MaaS Integration Changes (#20)

* update controller-manager image to support ceph

* install ceph-common on all nodes

* add upstream-dns to cleanup

* pull promenade images to ensure they're up to date

* force upstream dns configuration to be a string

* make armada-loader use host networking

This lets the pod come up right away regardless of network status.  This
just results in a cleaner startup, since kubelet tries to start it even
if network isn't ready if it sees the manifest.

* try to drain the node before exiting

This is slow, but safer when ceph is running.

* fix dnsmasq upstream configuration

cleanup.sh

@@ -2,17 +2,21 @@
 
 set -x
 
+export KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml
+kubectl drain --force $(hostname)
+
 systemctl stop kubelet
 docker rm -fv $(docker ps -aq)
 
 systemctl stop docker
 
-apt-get remove -qq -y dnsmasq
+apt-get remove -qq -y dnsmasq ceph-common
 
 systemctl daemon-reload
 
 rm -rf \
     /etc/dnsmasq.d/kubernetes-masters \
+    /etc/dnsmasq.d/upstream-dns \
     /etc/docker \
     /etc/kubernetes \
     /etc/systemd/system/docker.service.d \

genesis.sh

@@ -62,6 +62,7 @@ if [ -f "${PROMENADE_LOAD_IMAGE}" ]; then
   docker load -i "${PROMENADE_LOAD_IMAGE}"
 fi
 
+docker pull quay.io/attcomdev/promenade:experimental
 docker run -t --rm \
     --net host \
     -v /:/target \

join.sh

@@ -62,6 +62,7 @@ if [ -f "${PROMENADE_LOAD_IMAGE}" ]; then
   docker load -i "${PROMENADE_LOAD_IMAGE}"
 fi
 
+docker pull quay.io/attcomdev/promenade:experimental
 docker run -t --rm \
     -v /:/target \
     quay.io/attcomdev/promenade:experimental \

promenade/templates/common/etc/dnsmasq.d/upstream-dns

@@ -1,9 +1 @@
-no-hosts
-no-resolv
-
-strict-order
-
-server={{ config['Network']['cluster_dns'] }}
-{%- for dns_server in config['Network']['dns_servers'] %}
-server={{ dns_server }}
-{%- endfor %}
+server=/{{ config['Network']['cluster_domain'] }}/{{ config['Network']['cluster_dns'] }}

promenade/templates/common/usr/local/bin/bootstrap

@@ -4,7 +4,10 @@ set -ex
 
 export DEBIAN_FRONTEND=noninteractive
 
-apt-get install -y -qq --no-install-recommends dnsmasq socat
+apt-get install -y --no-install-recommends \
+    ceph-common \
+    dnsmasq \
+    socat
 
 systemctl daemon-reload
 systemctl enable kubelet

promenade/templates/genesis/etc/kubernetes/asset-loader/assets/kube-dns.yaml

@@ -7,7 +7,8 @@ metadata:
   labels:
     addonmanager.kubernetes.io/mode: EnsureExists
 data:
-  upstreamNameservers: {{ config['Network']['dns_servers'] | tojson }}
+  upstreamNameservers: |-
+    {{ config['Network']['dns_servers'] | tojson }}
 
 ---
 apiVersion: v1

promenade/templates/genesis/etc/kubernetes/kubelet/manifests/armada-loader.yaml

@@ -8,6 +8,7 @@ metadata:
     app: promenade
     component: armada-loader
 spec:
+  hostNetwork: true
   containers:
     - name: loader
       image: quay.io/attcomdev/armada:master

promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-controller-manager.yaml

@@ -11,13 +11,11 @@ metadata:
     scheduler.alpha.kubernetes.io/critical-pod: ''
 spec:
   hostNetwork: true
-  dnsPolicy: Default # Don't use cluster DNS.
   containers:
     - name: kube-controller-manager
-      image: gcr.io/google_containers/hyperkube-amd64:v1.6.4
+      image: quay.io/attcomdev/kube-controller-manager:v1.6.4
       command:
-      - ./hyperkube
-      - controller-manager
+      - kube-controller-manager
       - --allocate-node-cidrs=true
       - --cluster-cidr={{ config['Network']['pod_ip_cidr'] }}
       - --cluster-signing-cert-file=/etc/kubernetes/controller-manager/pki/cluster-ca.pem