Allow multiple join IPs

This provides more robustness in testing and removes a nosec.

Additionally, commit 5a8b1d8 introduced a random failure in the
resiliency gate, due to there being a chance to choose the intentionally
downed node for the join ip.

Change-Id: I77b410b8e51f9d41eca2be4f5f770694140733b4

promenade/config.py

@@ -165,6 +165,18 @@ class Configuration:
             if value:
                 return value
 
+    @property
+    def join_ips(self):
+        maybe_ips = self.get_path('KubernetesNode:join_ips')
+        if maybe_ips is not None:
+            return maybe_ips
+        else:
+            maybe_ip = self._get_first('KubernetesNode:join_ip', 'Genesis:ip')
+            if maybe_ip:
+                return [maybe_ip]
+            else:
+                return jinja2.StrictUndefined('Could not find join IPs')
+
     def get_path(self, path, default=None):
         kind, jsonpath = path.split(':')
         document = _get(self.documents, kind=kind)

promenade/control/join_scripts.py

@@ -14,7 +14,6 @@
 
 import falcon
 import kubernetes
-import random
 
 from promenade.control.base import BaseResource
 from promenade.builder import Builder
@@ -43,7 +42,7 @@ class JoinScriptsResource(BaseResource):
         dynamic_labels = _get_param_list(req, 'labels.dynamic')
         static_labels = _get_param_list(req, 'labels.static')
 
-        join_ip = _get_join_ip()
+        join_ips = _get_join_ips()
 
         try:
             config = Configuration.from_design_ref(
@@ -71,7 +70,7 @@ class JoinScriptsResource(BaseResource):
             'data': {
                 'hostname': hostname,
                 'ip': ip,
-                'join_ip': join_ip,
+                'join_ips': join_ips,
                 'labels': {
                     'dynamic': dynamic_labels,
                     'static': static_labels,
@@ -88,16 +87,13 @@ class JoinScriptsResource(BaseResource):
         resp.status = falcon.HTTP_200
 
 
-def _get_join_ip():
+def _get_join_ips():
     # TODO(mark-burnett): Handle errors
     kubernetes.config.load_incluster_config()
     client = kubernetes.client.CoreV1Api()
     response = client.list_node(label_selector='kubernetes-apiserver=enabled')
 
-    # Ignore bandit false positive: B311:blacklist
+    return list(map(_extract_ip, response.items))
-    # The choice of which master to join to is a load-balancing concern, not a
-    # security concern.
-    return random.choice(list(map(_extract_ip, response.items)))  # nosec
 
 
 def _extract_ip(item):

promenade/schemas/KubernetesNode.yaml

@@ -29,6 +29,11 @@ data:
     join_ip:
       $ref: '#/definitions/ip_address'
 
+    join_ips:
+      type: array
+      item:
+        $ref: '#/definitions/ip_address'
+
     labels:
       properties:
         static:
@@ -43,6 +48,5 @@ data:
 
   required:
     - ip
-    - join_ip
   additionalProperties: false
 ...

promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg

@@ -25,12 +25,12 @@ frontend default-kubernetes-fe
   bind *:{{ config['KubernetesNetwork:kubernetes.haproxy_port'] }}
   default_backend default-kubernetes-be
 
-{% set ip = config.get_first('KubernetesNode:join_ip', 'Genesis:ip') -%}
-
 backend default-kubernetes-be
   option tcp-check
   {%- set port = config['KubernetesNetwork:kubernetes.apiserver_port'] %}
+  {%- for ip in config.join_ips %}
   server s{{ ip }} {{ ip }}:{{ port }} check port {{ port }}
+  {%- endfor %}
 
 frontend kube-system-kubernetes-etcd-fe
   bind *:{{ config['KubernetesNetwork:etcd.haproxy_port'] }}
@@ -39,4 +39,6 @@ frontend kube-system-kubernetes-etcd-fe
 backend kube-system-kubernetes-etcd-be
   option tcp-check
   {%- set port = config['KubernetesNetwork:etcd.container_port'] %}
+  {%- for ip in config.join_ips %}
   server s{{ ip }} {{ ip }}:{{ port }} check port {{ port }}
+  {%- endfor %}

tools/gate/config-templates/joining-host-config.yaml

@@ -70,7 +70,9 @@ metadata:
 data:
   hostname: ${MASTER2_HOSTNAME}
   ip: ${MASTER2_IP}
-  join_ip: ${GENESIS_IP}
+  join_ips:
+    - ${GENESIS_IP}
+    - ${MASTER1_IP}
   labels:
     dynamic:
       - calico-etcd=enabled
@@ -100,7 +102,10 @@ metadata:
 data:
   hostname: ${WORKER_HOSTNAME}
   ip: ${WORKER_IP}
-  join_ip: ${MASTER1_IP}
+  join_ips:
+    - ${GENESIS_IP}
+    - ${MASTER1_IP}
+    - ${MASTER2_IP}
   labels:
     dynamic:
       - openstack-compute-node=enabled