diff --git a/promenade/config.py b/promenade/config.py index 25319a43..28e12008 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -165,6 +165,18 @@ class Configuration: if value: return value + @property + def join_ips(self): + maybe_ips = self.get_path('KubernetesNode:join_ips') + if maybe_ips is not None: + return maybe_ips + else: + maybe_ip = self._get_first('KubernetesNode:join_ip', 'Genesis:ip') + if maybe_ip: + return [maybe_ip] + else: + return jinja2.StrictUndefined('Could not find join IPs') + def get_path(self, path, default=None): kind, jsonpath = path.split(':') document = _get(self.documents, kind=kind) diff --git a/promenade/control/join_scripts.py b/promenade/control/join_scripts.py index 313d9e68..0d9283e2 100644 --- a/promenade/control/join_scripts.py +++ b/promenade/control/join_scripts.py @@ -14,7 +14,6 @@ import falcon import kubernetes -import random from promenade.control.base import BaseResource from promenade.builder import Builder @@ -43,7 +42,7 @@ class JoinScriptsResource(BaseResource): dynamic_labels = _get_param_list(req, 'labels.dynamic') static_labels = _get_param_list(req, 'labels.static') - join_ip = _get_join_ip() + join_ips = _get_join_ips() try: config = Configuration.from_design_ref( @@ -71,7 +70,7 @@ class JoinScriptsResource(BaseResource): 'data': { 'hostname': hostname, 'ip': ip, - 'join_ip': join_ip, + 'join_ips': join_ips, 'labels': { 'dynamic': dynamic_labels, 'static': static_labels, @@ -88,16 +87,13 @@ class JoinScriptsResource(BaseResource): resp.status = falcon.HTTP_200 -def _get_join_ip(): +def _get_join_ips(): # TODO(mark-burnett): Handle errors kubernetes.config.load_incluster_config() client = kubernetes.client.CoreV1Api() response = client.list_node(label_selector='kubernetes-apiserver=enabled') - # Ignore bandit false positive: B311:blacklist - # The choice of which master to join to is a load-balancing concern, not a - # security concern. - return random.choice(list(map(_extract_ip, response.items))) # nosec + return list(map(_extract_ip, response.items)) def _extract_ip(item): diff --git a/promenade/schemas/KubernetesNode.yaml b/promenade/schemas/KubernetesNode.yaml index 34a29571..13eb46e4 100644 --- a/promenade/schemas/KubernetesNode.yaml +++ b/promenade/schemas/KubernetesNode.yaml @@ -29,6 +29,11 @@ data: join_ip: $ref: '#/definitions/ip_address' + join_ips: + type: array + item: + $ref: '#/definitions/ip_address' + labels: properties: static: @@ -43,6 +48,5 @@ data: required: - ip - - join_ip additionalProperties: false ... diff --git a/promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg b/promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg index d6518775..715bc9fc 100644 --- a/promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg +++ b/promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg @@ -25,12 +25,12 @@ frontend default-kubernetes-fe bind *:{{ config['KubernetesNetwork:kubernetes.haproxy_port'] }} default_backend default-kubernetes-be -{% set ip = config.get_first('KubernetesNode:join_ip', 'Genesis:ip') -%} - backend default-kubernetes-be option tcp-check {%- set port = config['KubernetesNetwork:kubernetes.apiserver_port'] %} + {%- for ip in config.join_ips %} server s{{ ip }} {{ ip }}:{{ port }} check port {{ port }} + {%- endfor %} frontend kube-system-kubernetes-etcd-fe bind *:{{ config['KubernetesNetwork:etcd.haproxy_port'] }} @@ -39,4 +39,6 @@ frontend kube-system-kubernetes-etcd-fe backend kube-system-kubernetes-etcd-be option tcp-check {%- set port = config['KubernetesNetwork:etcd.container_port'] %} + {%- for ip in config.join_ips %} server s{{ ip }} {{ ip }}:{{ port }} check port {{ port }} + {%- endfor %} diff --git a/tools/gate/config-templates/joining-host-config.yaml b/tools/gate/config-templates/joining-host-config.yaml index ed83615f..d97c9ca9 100644 --- a/tools/gate/config-templates/joining-host-config.yaml +++ b/tools/gate/config-templates/joining-host-config.yaml @@ -70,7 +70,9 @@ metadata: data: hostname: ${MASTER2_HOSTNAME} ip: ${MASTER2_IP} - join_ip: ${GENESIS_IP} + join_ips: + - ${GENESIS_IP} + - ${MASTER1_IP} labels: dynamic: - calico-etcd=enabled @@ -100,7 +102,10 @@ metadata: data: hostname: ${WORKER_HOSTNAME} ip: ${WORKER_IP} - join_ip: ${MASTER1_IP} + join_ips: + - ${GENESIS_IP} + - ${MASTER1_IP} + - ${MASTER2_IP} labels: dynamic: - openstack-compute-node=enabled