From 7517d3161c84ed16ecaecd933fdfdea629f08318 Mon Sep 17 00:00:00 2001
From: Sean Eagan <sean.eagan@att.com>
Date: Tue, 14 May 2019 14:17:11 -0500
Subject: [PATCH] Fix potential port conflict

This change has been tested by the promenade resiliency gate.

This adds configuration for which ports to use for the tiller container
in the bootstrap-armada pod, and changes the defaults to be outside of
`net.ipv4.ip_local_port_range`, since the apiserver container in this pod
dynamically selects ports in that range to connect to etcd, which can
cause conflicts. See [0] for an example.

By default, since we're no longer using the standard tiller ports, this
does mean that we cannot connect to this tiller instance (before it's
replaced by the chart-based instance) via the helm CLI, until it supports
overriding the tiller port to connect to, however this should be
relatively soon [1].

[0]: https://github.com/helm/helm/issues/4886
[1]: https://github.com/helm/helm/pull/5590

Change-Id: Ief11411f079db27489e6974c028f6b7a16bb67bf
---
 doc/source/configuration/genesis.rst            |  3 +++
 promenade/schemas/Genesis.yaml                  |  8 ++++++++
 .../kubernetes/manifests/bootstrap-armada.yaml  | 17 ++++++++++++++---
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/doc/source/configuration/genesis.rst b/doc/source/configuration/genesis.rst
index 27e259e4..de22076b 100644
--- a/doc/source/configuration/genesis.rst
+++ b/doc/source/configuration/genesis.rst
@@ -25,6 +25,9 @@ Here is a complete sample document:
       ip: 192.168.77.10
       armada:
         target_manifest: cluster-bootstrap
+      tiller:
+        listen: 24134
+        probe_listen: 24135
       labels:
         static:
           - calico-etcd=enabled
diff --git a/promenade/schemas/Genesis.yaml b/promenade/schemas/Genesis.yaml
index c69a0cce..97ed03f5 100644
--- a/promenade/schemas/Genesis.yaml
+++ b/promenade/schemas/Genesis.yaml
@@ -99,6 +99,14 @@ data:
         auxiliary_threshold:
           type: integer
       additionalProperties: false
+    tiller:
+      type: object
+      properties:
+        listen:
+          type: integer
+        probe_listen:
+          type: integer
+      additionalProperties: false
 
     files:
       type: array
diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
index 12d90cb3..75557988 100644
--- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
+++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml
@@ -1,3 +1,9 @@
+{# By default, use tiller ports outside of `net.ipv4.ip_local_port_range` to
+   avoid conflicts with apiserver connections to etcd, see [0].
+   [0]: https://github.com/helm/helm/issues/4886
+#}
+{% set tiller_listen = config['Genesis:tiller.listen'] | default(24134) %}
+{% set tiller_probe_listen = config['Genesis:tiller.probe_listen'] | default(24135) %}
 ---
 apiVersion: v1
 kind: Pod
@@ -17,6 +23,10 @@ spec:
       image: {{ config['Genesis:images.helm.tiller'] }}
       command:
         - /tiller
+        - -listen
+        - ":{{ tiller_listen }}"
+        - -probe-listen
+        - ":{{ tiller_probe_listen }}"
         - -logtostderr
         - -v
         - "5"
@@ -25,7 +35,7 @@ spec:
         failureThreshold: 3
         httpGet:
           path: /liveness
-          port: 44135
+          port: {{ tiller_probe_listen }}
           scheme: HTTP
         initialDelaySeconds: 1
         periodSeconds: 10
@@ -33,14 +43,14 @@ spec:
         timeoutSeconds: 1
       name: tiller
       ports:
-      - containerPort: 44134
+      - containerPort: {{ tiller_listen }}
         name: tiller
         protocol: TCP
       readinessProbe:
         failureThreshold: 3
         httpGet:
           path: /readiness
-          port: 44135
+          port: {{ tiller_probe_listen }}
           scheme: HTTP
         initialDelaySeconds: 1
         periodSeconds: 10
@@ -65,6 +75,7 @@ spec:
                       apply \
                       --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
                       --tiller-host 127.0.0.1 \
+                      --tiller-port {{ tiller_listen }} \
                       /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
                   break
               fi