Use a separate etcd cluster for calico

This commit is contained in:
Mark Burnett 2017-07-06 10:00:24 -05:00
parent 3900378a54
commit f49c8d6834
25 changed files with 386 additions and 25 deletions

View File

@ -44,7 +44,7 @@ spec:
kube_service_ip: 10.96.0.1
pod_ip_cidr: 10.97.0.0/16
service_ip_cidr: 10.96.0.0/16
etcd_service_ip: 10.96.232.136
calico_etcd_service_ip: 10.96.232.136
dns_servers:
- 8.8.8.8
- 8.8.4.4

View File

@ -52,6 +52,14 @@ class Generator:
ca_name='etcd-peer',
cert_target='all',
key_target='masters')
calico_etcd_client_ca, calico_etcd_client_ca_key = keys.generate_ca(
ca_name='calico-etcd-client',
cert_target='all',
key_target='masters')
calico_etcd_peer_ca, calico_etcd_peer_ca_key = keys.generate_ca(
ca_name='calico-etcd-peer',
cert_target='all',
key_target='masters')
admin_cert, admin_cert_key = keys.generate_certificate(
name='admin',
@ -68,19 +76,27 @@ class Generator:
config.Configuration([
admin_cert,
admin_cert_key,
calico_etcd_client_ca,
calico_etcd_client_ca_key,
calico_etcd_peer_ca,
calico_etcd_peer_ca_key,
cluster_ca,
cluster_ca_key,
etcd_client_ca,
etcd_client_ca_key,
etcd_peer_ca,
etcd_peer_ca_key,
sa_pub,
sa_priv,
sa_pub,
]).write(os.path.join(output_dir, 'admin-bundle.yaml'))
complete_configuration = [
admin_cert,
admin_cert_key,
calico_etcd_client_ca,
calico_etcd_client_ca_key,
calico_etcd_peer_ca,
calico_etcd_peer_ca_key,
cluster_ca,
cluster_ca_key,
etcd_client_ca,
@ -89,8 +105,8 @@ class Generator:
etcd_peer_ca_key,
masters,
network,
sa_pub,
sa_priv,
sa_pub,
]
for hostname, data in cluster['nodes'].items():
@ -149,6 +165,8 @@ class Generator:
role_specific_documents.extend([
admin_cert,
admin_cert_key,
calico_etcd_client_ca,
calico_etcd_peer_ca,
cluster_ca_key,
etcd_client_ca,
etcd_peer_ca,
@ -158,8 +176,12 @@ class Generator:
if 'genesis' not in data.get('roles', []):
etcd_config = _master_etcd_config(
cluster_name, genesis_hostname, hostname, masters)
calico_etcd_config = _master_calico_etcd_config(
cluster_name, genesis_hostname, hostname, masters)
complete_configuration.append(etcd_config)
complete_configuration.append(calico_etcd_config)
role_specific_documents.append(etcd_config)
role_specific_documents.append(calico_etcd_config)
master_documents = _master_config(hostname, data,
masters, network, keys)
complete_configuration.extend(master_documents)
@ -169,6 +191,7 @@ class Generator:
role_specific_documents.extend(_genesis_config(hostname, data,
masters, network, keys))
role_specific_documents.append(_genesis_etcd_config(cluster_name, hostname))
role_specific_documents.append(_genesis_calico_etcd_config(cluster_name, hostname))
node.data['spec']['is_genesis'] = True
c = config.Configuration(common_documents + role_specific_documents)
@ -205,8 +228,23 @@ def _master_etcd_config(cluster_name, genesis_hostname, hostname, masters):
'auxiliary-etcd-0=https://%s:12380' % genesis_hostname,
'auxiliary-etcd-1=https://%s:22380' % genesis_hostname,
])
return _etcd_config(cluster_name, alias='master-etcd',
name='master-etcd:%s' % hostname,
return _etcd_config(cluster_name, alias='kube-etcd',
name='master-kube-etcd:%s' % hostname,
target=hostname,
initial_cluster=initial_cluster,
initial_cluster_state='existing')
def _master_calico_etcd_config(cluster_name, genesis_hostname, hostname, masters):
initial_cluster = ['%s=https://%s:6667' % (m['hostname'],
m['hostname'])
for m in masters['nodes']]
initial_cluster.extend([
'auxiliary-calico-etcd-0=https://%s:16667' % genesis_hostname,
'auxiliary-calico-etcd-1=https://%s:26667' % genesis_hostname,
])
return _etcd_config(cluster_name, alias='calico-etcd',
name='master-calico-etcd:%s' % hostname,
target=hostname,
initial_cluster=initial_cluster,
initial_cluster_state='existing')
@ -218,8 +256,21 @@ def _genesis_etcd_config(cluster_name, hostname):
'auxiliary-etcd-0=https://%s:12380' % hostname,
'auxiliary-etcd-1=https://%s:22380' % hostname,
]
return _etcd_config(cluster_name, alias='genesis-etcd',
name='master-etcd:%s' % hostname,
return _etcd_config(cluster_name, alias='kube-etcd',
name='master-kube-etcd:%s' % hostname,
target=hostname,
initial_cluster=initial_cluster,
initial_cluster_state='new')
def _genesis_calico_etcd_config(cluster_name, hostname):
initial_cluster = [
'%s=https://%s:6667' % (hostname, hostname),
'auxiliary-calico-etcd-0=https://%s:16667' % hostname,
'auxiliary-calico-etcd-1=https://%s:26667' % hostname,
]
return _etcd_config(cluster_name, alias='calico-etcd',
name='master-calico-etcd:%s' % hostname,
target=hostname,
initial_cluster=initial_cluster,
initial_cluster_state='new')
@ -256,7 +307,7 @@ def _master_config(hostname, host_data, masters, network, keys):
'calico-etcd.kube-system',
'calico-etcd.kube-system.svc',
'calico-etcd.kube-system.svc.cluster.local',
network['etcd_service_ip'],
network['calico_etcd_service_ip'],
]
docs = []
@ -284,6 +335,22 @@ def _master_config(hostname, host_data, masters, network, keys):
target=hostname,
))
docs.extend(keys.generate_certificate(
alias='calico-etcd-client',
name='calico-etcd:client:%s' % hostname,
ca_name='calico-etcd-client',
hosts=kube_domains + calico_domains + [hostname, host_data['ip']],
target=hostname,
))
docs.extend(keys.generate_certificate(
alias='calico-etcd-peer',
name='calico-etcd:peer:%s' % hostname,
ca_name='calico-etcd-peer',
hosts=kube_domains + [hostname, host_data['ip']],
target=hostname,
))
docs.extend(keys.generate_certificate(
alias='apiserver',
name='apiserver:%s' % hostname,
@ -341,10 +408,24 @@ def _genesis_config(hostname, host_data, masters, network, keys):
target=hostname,
))
docs.extend(keys.generate_certificate(
name='auxiliary-calico-etcd-%d-client' % i,
ca_name='calico-etcd-client',
hosts=[hostname, host_data['ip']],
target=hostname,
))
docs.extend(keys.generate_certificate(
name='auxiliary-calico-etcd-%d-peer' % i,
ca_name='calico-etcd-peer',
hosts=[hostname, host_data['ip']],
target=hostname,
))
docs.extend(keys.generate_certificate(
alias='etcd-calico-client',
name='etcd:client:calico',
ca_name='etcd-client',
alias='calico-etcd-node-client',
name='calico-etcd:node',
ca_name='calico-etcd-client',
target=hostname,
))

View File

@ -6,19 +6,19 @@ kind: Service
metadata:
labels:
tier: control-plane
component: kube-etcd
component: calico-etcd
name: calico-etcd
namespace: kube-system
spec:
# Select the calico-etcd pod running on the master.
selector:
tier: control-plane
component: kube-etcd
component: calico-etcd
# This ClusterIP needs to be known in advance, since we cannot rely
# on DNS to get access to etcd.
clusterIP: {{ config['Network']['etcd_service_ip'] }}
clusterIP: {{ config['Network']['calico_etcd_service_ip'] }}
ports:
- port: 2379
- port: 6666
---
# Calico Version v2.2.1
@ -37,7 +37,7 @@ metadata:
data:
# The location of your etcd cluster. This uses the Service clusterIP
# defined below.
etcd_endpoints: https://{{ config['Network']['etcd_service_ip'] }}:2379
etcd_endpoints: https://{{ config['Network']['calico_etcd_service_ip'] }}:6666
# Configure the Calico backend to use.
calico_backend: "bird"
@ -96,9 +96,9 @@ data:
# not using TLS for etcd.
# This self-hosted install expects three files with the following names. The values
# should be base64 encoded strings of the entire contents of each file.
etcd-key: {{ config.get(kind='CertificateKey', alias='etcd-calico-client')['data'] | b64enc }}
etcd-cert: {{ config.get(kind='Certificate', alias='etcd-calico-client')['data'] | b64enc }}
etcd-ca: {{ config.get(kind='CertificateAuthority', name='etcd-client')['data'] | b64enc }}
etcd-key: {{ config.get(kind='CertificateKey', alias='calico-etcd-node-client')['data'] | b64enc }}
etcd-cert: {{ config.get(kind='Certificate', alias='calico-etcd-node-client')['data'] | b64enc }}
etcd-ca: {{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] | b64enc }}
---
# This manifest installs the calico/node container, as well

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-0-client')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-0-client')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-0-peer')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-0-peer')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateAuthority', name='calico-etcd-peer')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-1-client')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-1-client')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-1-peer')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-1-peer')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateAuthority', name='calico-etcd-peer')['data'] }}

View File

@ -0,0 +1,194 @@
---
apiVersion: v1
kind: Pod
metadata:
name: auxiliary-calico-etcd
namespace: kube-system
labels:
component: auxiliary-calico-etcd
promenade: genesis
spec:
hostNetwork: true
containers:
- name: auxiliary-calico-etcd-0
image: quay.io/coreos/etcd:v3.0.17
env:
- name: ETCD_NAME
value: auxiliary-calico-etcd-0
- name: ETCD_CLIENT_CERT_AUTH
value: "true"
- name: ETCD_PEER_CLIENT_CERT_AUTH
value: "true"
- name: ETCD_DATA_DIR
value: /var/lib/auxiliary-calico-etcd-0
- name: ETCD_TRUSTED_CA_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/client-ca.pem
- name: ETCD_CERT_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client.pem
- name: ETCD_KEY_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client-key.pem
- name: ETCD_PEER_TRUSTED_CA_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/peer-ca.pem
- name: ETCD_PEER_CERT_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer.pem
- name: ETCD_PEER_KEY_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer-key.pem
- name: ETCD_ADVERTISE_CLIENT_URLS
value: https://{{ config['Node']['hostname'] }}:16666
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
value: https://{{ config['Node']['hostname'] }}:16667
- name: ETCD_INITIAL_CLUSTER_TOKEN
value: promenade-calico-etcd-token
- name: ETCD_LISTEN_CLIENT_URLS
value: https://0.0.0.0:16666
- name: ETCD_LISTEN_PEER_URLS
value: https://0.0.0.0:16667
- name: ETCD_INITIAL_CLUSTER_STATE
value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster_state'] }}
- name: ETCD_INITIAL_CLUSTER
value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster'] | join(',') }}
ports:
- name: client
containerPort: 16666
- name: peer
containerPort: 16667
resources:
limits:
cpu: 100m
requests:
cpu: 100m
volumeMounts:
- name: data-0
mountPath: /var/lib/auxiliary-calico-etcd-0
- name: pki-0
mountPath: /etc/kubernetes/auxiliary-calico-etcd-0/pki
readOnly: true
- name: auxiliary-calico-etcd-1
image: quay.io/coreos/etcd:v3.0.17
env:
- name: ETCD_NAME
value: auxiliary-calico-etcd-1
- name: ETCD_CLIENT_CERT_AUTH
value: "true"
- name: ETCD_PEER_CLIENT_CERT_AUTH
value: "true"
- name: ETCD_DATA_DIR
value: /var/lib/auxiliary-calico-etcd-1
- name: ETCD_TRUSTED_CA_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/client-ca.pem
- name: ETCD_CERT_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client.pem
- name: ETCD_KEY_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client-key.pem
- name: ETCD_PEER_TRUSTED_CA_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/peer-ca.pem
- name: ETCD_PEER_CERT_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer.pem
- name: ETCD_PEER_KEY_FILE
value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer-key.pem
- name: ETCD_ADVERTISE_CLIENT_URLS
value: https://{{ config['Node']['hostname'] }}:26666
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
value: https://{{ config['Node']['hostname'] }}:26667
- name: ETCD_INITIAL_CLUSTER_TOKEN
value: promenade-calico-etcd-token
- name: ETCD_LISTEN_CLIENT_URLS
value: https://0.0.0.0:26666
- name: ETCD_LISTEN_PEER_URLS
value: https://0.0.0.0:26667
- name: ETCD_INITIAL_CLUSTER_STATE
value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster_state'] }}
- name: ETCD_INITIAL_CLUSTER
value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster'] | join(',') }}
ports:
- name: client
containerPort: 26666
- name: peer
containerPort: 26667
resources:
limits:
cpu: 100m
requests:
cpu: 100m
volumeMounts:
- name: data-1
mountPath: /var/lib/auxiliary-calico-etcd-1
- name: pki-1
mountPath: /etc/kubernetes/auxiliary-calico-etcd-1/pki
readOnly: true
- name: cluster-monitor
image: quay.io/coreos/etcd:v3.0.17
command:
- sh
- -c
- |-
set -x
while true; do
if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -ge {{ config['Masters']['nodes'] | length }} ]; then
{%- for master in config['Masters']['nodes'] %}
etcdctl member add {{ master['hostname'] }} --peer-urls https://{{ master['hostname'] }}:6667
{%- endfor %}
break
fi
done
while true; do
sleep 5
if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -eq {{ 2 + (config['Masters']['nodes'] | length) }} ]; then
etcdctl member remove $(etcdctl member list | grep auxiliary-calico-etcd-1 | cut -d , -f 1)
etcdctl member remove $(etcdctl member list | grep auxiliary-calico-etcd-0 | cut -d , -f 1)
sleep 60
rm -rf \
/var/lib/auxiliary-calico-etcd-0 \
/var/lib/auxiliary-calico-etcd-1 \
/etc/kubernetes/auxiliary-calico-etcd-0 \
/etc/kubernetes/auxiliary-calico-etcd-1 \
/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml
sleep 10000
fi
done
resources:
limits:
cpu: 100m
requests:
cpu: 100m
env:
- name: ETCDCTL_API
value: "3"
- name: ETCDCTL_CACERT
value: /etc/kubernetes/calico-etcd/pki/client-ca.pem
- name: ETCDCTL_CERT
value: /etc/kubernetes/calico-etcd/pki/etcd-client.pem
- name: ETCDCTL_ENDPOINTS
value: https://{{ config['Node']['ip'] }}:6666
- name: ETCDCTL_KEY
value: /etc/kubernetes/calico-etcd/pki/etcd-client-key.pem
volumeMounts:
- name: pki
mountPath: /etc/kubernetes/calico-etcd/pki
readOnly: true
- name: manifests
mountPath: /etc/kubernetes/kubelet/manifests
- name: varlib
mountPath: /var/lib
volumes:
- name: data-0
hostPath:
path: /var/lib/auxiliary-calico-etcd-0
- name: data-1
hostPath:
path: /var/lib/auxiliary-calico-etcd-1
- name: pki
hostPath:
path: /etc/kubernetes/calico-etcd/pki
- name: pki-0
hostPath:
path: /etc/kubernetes/auxiliary-calico-etcd-0/pki
- name: pki-1
hostPath:
path: /etc/kubernetes/auxiliary-calico-etcd-1/pki
- name: manifests
hostPath:
path: /etc/kubernetes/kubelet/manifests
- name: varlib
hostPath:
path: /var/lib

View File

@ -44,9 +44,9 @@ spec:
- name: ETCD_LISTEN_PEER_URLS
value: https://0.0.0.0:12380
- name: ETCD_INITIAL_CLUSTER_STATE
value: {{ config['Etcd']['initial_cluster_state'] }}
value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster_state'] }}
- name: ETCD_INITIAL_CLUSTER
value: {{ config['Etcd']['initial_cluster'] | join(',') }}
value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster'] | join(',') }}
ports:
- name: client
containerPort: 12379
@ -97,9 +97,9 @@ spec:
- name: ETCD_LISTEN_PEER_URLS
value: https://0.0.0.0:22380
- name: ETCD_INITIAL_CLUSTER_STATE
value: {{ config['Etcd']['initial_cluster_state'] }}
value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster_state'] }}
- name: ETCD_INITIAL_CLUSTER
value: {{ config['Etcd']['initial_cluster'] | join(',') }}
value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster'] | join(',') }}
ports:
- name: client
containerPort: 22379

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateKey', alias='calico-etcd-client')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='Certificate', alias='calico-etcd-client')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateKey', alias='calico-etcd-peer')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='Certificate', alias='calico-etcd-peer')['data'] }}

View File

@ -0,0 +1 @@
{{ config.get(kind='CertificateAuthority', name='calico-etcd-peer')['data'] }}

View File

@ -0,0 +1,68 @@
---
apiVersion: v1
kind: Pod
metadata:
name: calico-etcd
namespace: kube-system
labels:
tier: control-plane
component: calico-etcd
spec:
hostNetwork: true
containers:
- name: k8s-etcd
image: quay.io/coreos/etcd:v3.0.17
env:
- name: ETCD_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: ETCD_CLIENT_CERT_AUTH
value: "true"
- name: ETCD_PEER_CLIENT_CERT_AUTH
value: "true"
- name: ETCD_DATA_DIR
value: /var/lib/calico-etcd
- name: ETCD_TRUSTED_CA_FILE
value: /etc/kubernetes/calico-etcd/pki/client-ca.pem
- name: ETCD_CERT_FILE
value: /etc/kubernetes/calico-etcd/pki/etcd-client.pem
- name: ETCD_KEY_FILE
value: /etc/kubernetes/calico-etcd/pki/etcd-client-key.pem
- name: ETCD_PEER_TRUSTED_CA_FILE
value: /etc/kubernetes/calico-etcd/pki/peer-ca.pem
- name: ETCD_PEER_CERT_FILE
value: /etc/kubernetes/calico-etcd/pki/etcd-peer.pem
- name: ETCD_PEER_KEY_FILE
value: /etc/kubernetes/calico-etcd/pki/etcd-peer-key.pem
- name: ETCD_ADVERTISE_CLIENT_URLS
value: https://$(ETCD_NAME):6666
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
value: https://$(ETCD_NAME):6667
- name: ETCD_INITIAL_CLUSTER_TOKEN
value: promenade-calico-etcd-token
- name: ETCD_LISTEN_CLIENT_URLS
value: https://0.0.0.0:6666
- name: ETCD_LISTEN_PEER_URLS
value: https://0.0.0.0:6667
- name: ETCD_INITIAL_CLUSTER_STATE
value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster_state'] }}
- name: ETCD_INITIAL_CLUSTER
value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster'] | join(',') }}
ports:
- name: client
containerPort: 6666
- name: peer
containerPort: 6667
volumeMounts:
- name: data
mountPath: /var/lib/calico-etcd
- name: pki
mountPath: /etc/kubernetes/calico-etcd/pki
volumes:
- name: data
hostPath:
path: /var/lib/calico-etcd
- name: pki
hostPath:
path: /etc/kubernetes/calico-etcd/pki

View File

@ -46,9 +46,9 @@ spec:
- name: ETCD_LISTEN_PEER_URLS
value: https://0.0.0.0:2380
- name: ETCD_INITIAL_CLUSTER_STATE
value: {{ config['Etcd']['initial_cluster_state'] }}
value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster_state'] }}
- name: ETCD_INITIAL_CLUSTER
value: {{ config['Etcd']['initial_cluster'] | join(',') }}
value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster'] | join(',') }}
ports:
- name: client
containerPort: 2379