Use HAProxy for apiserver discovery

This removes the reliance on coredns for APIserver discovery, allowing
a simpler configuration that is compatible with corednx 1.0.x

Change-Id: Ia3b7b5627c16ec47af6b0d6d5e8dee2674e9b1ee
This commit is contained in:
Mark Burnett 2018-01-30 23:34:08 -06:00
parent 753576a89b
commit ff3787c2ad
48 changed files with 1121 additions and 655 deletions

View File

@ -1,130 +0,0 @@
#!/bin/sh
{{- $envAll := . }}
set -x
export MANIFEST_PATH=/host{{ .Values.anchor.kubelet.manifest_path }}/{{ .Values.service.name }}.yaml
export ETC_PATH=/host{{ .Values.coredns.host_etc_path }}
TOKEN_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token
CA_CERT_PATH=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
copy_etc_files() {
mkdir -p $ETC_PATH/zones
cp /configmap/* /secret/* $ETC_PATH
create_corefile
}
create_corefile() {
cat <<EOCOREFILE > $ETC_PATH/Corefile
promenade {
file /etc/coredns/zones/promenade
loadbalance
errors stdout
log stdout
}
. {
kubernetes{{- range .Values.coredns.kubernetes_zones }} {{ . -}}{{- end }} {
endpoint https://{{ .Values.network.kubernetes_netloc }}
tls /etc/coredns/coredns.pem /etc/coredns/coredns-key.pem /etc/coredns/cluster-ca.pem
pods insecure
}
{{- if .Values.coredns.upstream_nameservers }}
{{ range .Values.coredns.upstream_nameservers }}
proxy . {{ . }}
{{- end }}
{{- end }}
loadbalance
cache {{ .Values.coredns.cache.ttl }}
errors stdout
log stdout
}
EOCOREFILE
}
create_manifest() {
mkdir -p $(dirname $MANIFEST_PATH)
# XXX liveness/readiness probes
cat <<EODOC > $MANIFEST_PATH
---
apiVersion: v1
kind: Pod
metadata:
name: {{ .Values.service.name }}
namespace: {{ .Release.Namespace }}
labels:
{{ .Values.service.name }}-service: enabled
anchor-managed: enabled
spec:
hostNetwork: true
containers:
- name: coredns
image: {{ .Values.images.coredns }}
command:
- /coredns
- -conf
- /etc/coredns/Corefile
volumeMounts:
- name: etc
mountPath: /etc/coredns
volumes:
- name: etc
hostPath:
path: {{ .Values.coredns.host_etc_path }}
EODOC
}
update_managed_zones() {
{{- range .Values.coredns.zones }}
FILENAME="$ETC_PATH/zones/{{ .name }}"
NEXT_FILENAME="${FILENAME}-next"
SUCCESS=1
NOW=$(date +%s)
# Add Header
cat <<EOBIND > $NEXT_FILENAME
\$ORIGIN {{ .name }}.
{{ .name }}. IN SOA @ root $NOW 3h 15m 1w 1d
EOBIND
{{ range .services }}
# Don't accidentally log service account token
set +x
SERVICE_IPS=$(kubectl \
--server https://{{ $envAll.Values.network.kubernetes_netloc }} \
--certificate-authority $CA_CERT_PATH \
--token $(cat $TOKEN_PATH) \
-n {{ .service.namespace }} \
get ep {{ .service.name }} \
-o 'jsonpath={.subsets[*].addresses[*].ip}')
set -x
if [ "x$SERVICE_IPS" != "x" ]; then
for IP in $SERVICE_IPS; do
echo {{ .bind_name }} IN A $IP >> $NEXT_FILENAME
done
else
echo Failed to upate zone file for {{ .name }}
SUCCESS=0
fi
{{- end }}
if [ $SUCCESS = 1 ]; then
echo Replacing zone file $FILENAME
mv $NEXT_FILENAME $FILENAME
fi
{{- end }}
}
copy_etc_files
create_manifest
while true; do
update_managed_zones
sleep {{ .Values.anchor.period }}
done

View File

@ -1,8 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .Values.service.name }}-bin
data:
anchor: |+
{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

View File

@ -4,5 +4,4 @@ kind: ConfigMap
metadata:
name: {{ .Values.service.name }}-etc
data:
cluster-ca.pem: {{ .Values.tls.ca | quote }}
coredns.pem: {{ .Values.tls.cert | quote }}
Corefile: {{ .Values.conf.coredns.corefile | quote }}

View File

@ -1,83 +0,0 @@
---
apiVersion: "extensions/v1beta1"
kind: DaemonSet
metadata:
name: {{ .Values.service.name }}-anchor
labels:
application: coredns
component: coredns-anchor
spec:
selector:
matchLabels:
{{ .Values.service.name | quote }}: anchor
updateStrategy:
rollingUpdate:
maxUnavailable: 1
template:
metadata:
labels:
{{ .Values.service.name | quote }}: anchor
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
spec:
hostNetwork: true
{{- if .Values.node_selector.key }}
nodeSelector:
{{ .Values.node_selector.key }}: {{ .Values.node_selector.value }}
{{- end }}
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
- key: CriticalAddonsOnly
operator: Exists
serviceAccountName: {{ .Values.service.name }}
containers:
- name: anchor
image: {{ .Values.images.anchor }}
command:
- /tmp/bin/anchor
# livenessProbe:
# httpGet:
# path: /healthz
# port: 10249
# initialDelaySeconds: 15
# periodSeconds: 15
# failureThreshold: 3
# readinessProbe:
# exec:
# command:
# - sh
# - -c
# - |-
# set -ex
#
# iptables-save | grep 'default/kubernetes:https'
# initialDelaySeconds: 15
# periodSeconds: 15
volumeMounts:
- name: bin
mountPath: /tmp/bin
- name: etc
mountPath: /configmap
- name: host
mountPath: /host
- name: secret
mountPath: /secret
volumes:
- name: bin
configMap:
name: {{ .Values.service.name }}-bin
defaultMode: 0555
- name: etc
configMap:
name: {{ .Values.service.name }}-etc
defaultMode: 0444
- name: host
hostPath:
path: /
- name: secret
secret:
secretName: {{ .Values.service.name }}
defaultMode: 0444

View File

@ -0,0 +1,85 @@
{{/*
Copyright 2018 AT&T Intellectual Property. All other rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: coredns
labels:
{{ .Values.service.name }}: enabled
kubernetes.io/name: "CoreDNS"
spec:
replicas: 2
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
selector:
matchLabels:
{{ .Values.service.name }}: enabled
template:
metadata:
labels:
{{ .Values.service.name }}: enabled
spec:
serviceAccountName: coredns
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: {{ .Values.service.name }}
operator: In
values:
- enabled
topologyKey: kubernetes.io/hostname
containers:
- name: coredns
image: {{ .Values.images.tags.coredns | quote }}
imagePullPolicy: {{ .Values.images.pull_policy | quote }}
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
dnsPolicy: Default
volumes:
- name: config-volume
configMap:
name: {{ .Values.service.name }}-etc
items:
- key: Corefile
path: Corefile

View File

@ -3,7 +3,7 @@ apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
namespace: {{ .Release.Namespace }}
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
@ -17,7 +17,7 @@ subjects:
apiGroup: rbac.authorization.k8s.io
- kind: ServiceAccount
name: coredns
namespace: kube-system
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: system:coredns

View File

@ -1,8 +0,0 @@
---
apiVersion: v1
kind: Secret
metadata:
name: {{ .Values.service.name }}
type: Opaque
data:
coredns-key.pem: {{ .Values.tls.key | b64enc }}

View File

@ -3,9 +3,12 @@ apiVersion: v1
kind: Service
metadata:
name: {{ .Values.service.name }}
labels:
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "CoreDNS"
spec:
selector:
{{ .Values.service.name }}-service: enabled
{{ .Values.service.name }}: enabled
clusterIP: {{ .Values.service.ip }}
ports:
- name: dns

View File

@ -1,38 +1,26 @@
anchor:
kubelet:
manifest_path: /etc/kubernetes/manifests
node_selector: {}
period: 30
tls:
ca: placeholder
cert: placeholder
key: placeholder
coredns:
kubernetes_zones:
- cluster.local
cache:
ttl: 60
host_etc_path: /etc/coredns
upstream_nameservers:
- 8.8.8.8
- 8.8.4.4
zones:
- name: promenade
services:
- bind_name: apiserver.kubernetes
service:
name: kubernetes
namespace: default
conf:
coredns:
corefile: |
.:53 {
errors
health
autopath @kubernetes
kubernetes cluster.local 10.96.0.0/16 10.97.0.0/16 {
pods insecure
fallthrough in-addr.arpa ip6.arpa
upstream 8.8.8.8
upstream 8.8.4.4
}
prometheus :9153
proxy . 8.8.8.8
proxy . 8.8.4.4
cache 30
}
images:
anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
coredns: coredns/coredns:0.9.9
network:
kubernetes_netloc: 10.96.0.1
tags:
coredns: coredns/coredns:1.0.5
pull_policy: "IfNotPresent"
node_selector: {}

View File

@ -0,0 +1,4 @@
apiVersion: v1
description: A chart for using HAProxy for Kubernetes API server discovery
name: haproxy
version: 0.1.0

View File

@ -0,0 +1,4 @@
dependencies:
- name: helm-toolkit
repository: http://localhost:8879/charts
version: 0.1.0

View File

@ -0,0 +1,129 @@
#!/bin/sh
{{/*
Copyright 2018 AT&T Intellectual Property. All other rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- $envAll := . }}
set -x
compare_copy_files() {
{{- range .Values.conf.anchor.files_to_copy }}
if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
mkdir -p $(dirname /host{{ .dest }})
cp {{ .source }} /host{{ .dest }}
fi
{{- end }}
}
install_config() {
SUCCESS=1
# Inject global and default config
mkdir -p $(dirname "$HAPROXY_CONF")
cp "$HAPROXY_HEADER" "$NEXT_HAPROXY_CONF"
{{- range $namespace, $services := $envAll.Values.conf.anchor.services }}
{{- range $service, $svc_data := $services }}
echo Constructing config for namespace=\"{{ $namespace }}\" service=\"{{ $service }}\"
# NOTE(mark-burnett): Don't accidentally log service account token.
set +x
SERVICE_IPS=$(kubectl \
--server "$KUBE_URL" \
--certificate-authority "$KUBE_CA" \
--token $(cat "$KUBE_TOKEN") \
--namespace {{ $namespace }} \
get endpoints {{ $service }} \
-o 'jsonpath={.subsets[0].addresses[*].ip}')
DEST_PORT=$(kubectl \
--server "$KUBE_URL" \
--certificate-authority "$KUBE_CA" \
--token $(cat "$KUBE_TOKEN") \
--namespace {{ $namespace }} \
get endpoints {{ $service }} \
-o 'jsonpath={.subsets[0].ports[0].port}')
set -x
if [ "x$SERVICE_IPS" != "x" ]; then
if [ "x$DEST_PORT" != "x" ]; then
IDENTIFIER=$(echo "{{ $namespace }}-{{ $service }}")
# Add frontend config
echo >> "$NEXT_HAPROXY_CONF"
echo "frontend ${IDENTIFIER}-fe" >> "$NEXT_HAPROXY_CONF"
{{- range $envAll.Values.conf.haproxy.conf_parts.frontend }}
echo " {{ . }}" >> "$NEXT_HAPROXY_CONF"
{{- end }}
{{- range $svc_data.conf_parts.frontend }}
echo " {{ . }}" >> "$NEXT_HAPROXY_CONF"
{{- end }}
echo " default_backend ${IDENTIFIER}-be" >> "$NEXT_HAPROXY_CONF"
# Add backend config
echo >> "$NEXT_HAPROXY_CONF"
echo "backend ${IDENTIFIER}-be" >> "$NEXT_HAPROXY_CONF"
{{- range $envAll.Values.conf.haproxy.conf_parts.backend }}
echo " {{ . }}" >> "$NEXT_HAPROXY_CONF"
{{- end }}
{{- range $svc_data.conf_parts.backend }}
echo " {{ . }}" >> "$NEXT_HAPROXY_CONF"
{{- end }}
for IP in $SERVICE_IPS; do
echo " server s$IP $IP:$DEST_PORT" {{ $svc_data.server_opts | quote }} >> "$NEXT_HAPROXY_CONF"
done
else
echo Failed to get destination port for service.
SUCCESS=0
fi
else
echo Failed to get endpoint IPs for service.
SUCCESS=0
fi
{{- end }}
{{- end }}
if [ $SUCCESS = 1 ]; then
mkdir -p $(dirname "$HAPROXY_CONF")
if ! cmp -s "$HAPROXY_CONF" "$NEXT_HAPROXY_CONF"; then
echo Replacing HAProxy config file "$HAPROXY_CONF" with:
cat "$NEXT_HAPROXY_CONF"
echo
mv "$NEXT_HAPROXY_CONF" "$HAPROXY_CONF"
else
echo HAProxy config file unchanged.
fi
fi
}
cleanup() {
{{- range .Values.conf.anchor.files_to_copy }}
rm -f /host{{ .dest }}
{{- end }}
rm -f "$HAPROXY_CONF" "$NEXT_HAPROXY_CONF"
}
while true; do
if [ -e /tmp/stop ]; then
echo Stopping
cleanup
break
fi
install_config
compare_copy_files
sleep {{ .Values.conf.anchor.period }}
done

View File

@ -0,0 +1,21 @@
#!/bin/sh
{{/*
Copyright 2018 AT&T Intellectual Property. All other rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -x
touch /tmp/stop
sleep {{ .Values.conf.anchor.period }}

View File

@ -0,0 +1,25 @@
{{/*
Copyright 2018 AT&T Intellectual Property. All other rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: haproxy-bin
data:
anchor.sh: |
{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
pre_stop.sh: |
{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

View File

@ -0,0 +1,35 @@
{{/*
Copyright 2018 AT&T Intellectual Property. All other rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: haproxy-etc
data:
haproxy.cfg.header: |
global
{{- range .Values.conf.haproxy.conf_parts.global }}
{{ . }}
{{- end }}
stats socket /tmp/haproxy.sock mode 700 level admin expose-fd listeners
defaults
{{- range .Values.conf.haproxy.conf_parts.defaults }}
{{ . }}
{{- end }}
haproxy.yaml: |
{{ tuple "etc/_haproxy.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

View File

@ -0,0 +1,81 @@
{{/*
Copyright 2018 AT&T Intellectual Property. All other rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- $envAll := . }}
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: haproxy-anchor
spec:
{{ tuple $envAll "haproxy_anchor" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
template:
metadata:
labels:
{{ tuple $envAll "kubernetes" "haproxy-anchor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
- key: CriticalAddonsOnly
operator: Exists
terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.haproxy_anchor.timeout }}
serviceAccountName: haproxy-anchor
containers:
- name: anchor
image: {{ .Values.images.tags.anchor }}
imagePullPolicy: {{ .Values.images.pull_policy }}
env:
- name: HAPROXY_HEADER
value: /tmp/etc/haproxy.cfg.header
- name: HAPROXY_CONF
value: /host{{ .Values.conf.haproxy.host_config_dir }}/haproxy.cfg
- name: NEXT_HAPROXY_CONF
value: "$(HAPROXY_CONF)-next"
- name: HAPROXY_SERVER_OPTS
value: {{ .Values.conf.haproxy.server_opts | quote }}
- name: KUBE_CA
value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
- name: KUBE_TOKEN
value: /var/run/secrets/kubernetes.io/serviceaccount/token
- name: KUBE_URL
value: {{ .Values.conf.anchor.kubernetes_url | quote }}
command:
- /tmp/bin/anchor.sh
lifecycle:
preStop:
exec:
command:
- /tmp/bin/pre_stop.sh
volumeMounts:
- name: haproxy-bin
mountPath: /tmp/bin
- name: haproxy-etc
mountPath: /tmp/etc
- name: host
mountPath: /host
volumes:
- name: haproxy-bin
configMap:
name: haproxy-bin
defaultMode: 0555
- name: haproxy-etc
configMap:
name: haproxy-etc
defaultMode: 0444
- name: host
hostPath:
path: /

View File

@ -0,0 +1,88 @@
{{/*
Copyright 2018 AT&T Intellectual Property. All other rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
---
apiVersion: v1
kind: Pod
metadata:
name: haproxy
namespace: {{ .Release.Namespace }}
spec:
hostNetwork: true
containers:
- name: haproxy
image: {{ .Values.images.tags.haproxy }}
imagePullPolicy: {{ .Values.images.pull_policy }}
hostNetwork: true
env:
- name: HAPROXY_CONF
value: {{ .Values.conf.haproxy.container_config_dir }}/haproxy.cfg
- name: LIVE_HAPROXY_CONF
value: /tmp/live_haproxy.cfg
command:
- /bin/sh
- -c
- |
set -eux
while [ ! -s "$HAPROXY_CONF" ]; do
echo Waiting for "HAPROXY_CONF"
sleep 1
done
echo vvv Starting with initial config vvv
cat "$HAPROXY_CONF"
echo
cp "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"
# NOTE(mark-burnett): sleep for clearer log output
sleep 1
haproxy -D -f "$LIVE_HAPROXY_CONF" -p /tmp/haproxy.pid
echo HAProxy started, monitoring for config changes..
set +x
while true; do
if ! cmp -s "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"; then
echo vvv Replacing old config vvv
cat "$LIVE_HAPROXY_CONF"
echo
echo vvv With new config vvv
cat "$HAPROXY_CONF"
echo
cat "$HAPROXY_CONF" > "$LIVE_HAPROXY_CONF"
# NOTE(mark-burnett): sleep for clearer log output
sleep 1
set -x
haproxy -D -f "$LIVE_HAPROXY_CONF" -p /tmp/haproxy.pid \
-x /tmp/haproxy.sock \
-sf $(cat /tmp/haproxy.pid)
set +x
fi
sleep {{ .Values.conf.haproxy.period }}
done
volumeMounts:
- name: etc
mountPath: {{ .Values.conf.haproxy.container_config_dir }}
readOnly: True
volumes:
- name: etc
hostPath:
path: {{ .Values.conf.haproxy.host_config_dir }}

View File

@ -0,0 +1,52 @@
{{/*
Copyright 2018 AT&T Intellectual Property. All other rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- $envAll := . }}
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: haproxy-anchor
{{- range $namespace, $services := $envAll.Values.conf.anchor.services }}
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: haproxy-anchor
namespace: {{ $namespace }}
rules:
- apiGroups: [""]
resources:
- endpoints
verbs:
- get
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: haproxy-anchor
namespace: {{ $namespace }}
subjects:
- kind: ServiceAccount
name: haproxy-anchor
namespace: {{ $envAll.Release.Namespace }}
apiGroup: ""
roleRef:
kind: Role
name: haproxy-anchor
apiGroup: rbac.authorization.k8s.io
{{- end }}

View File

@ -0,0 +1,88 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
conf:
anchor:
files_to_copy:
- source: /tmp/etc/haproxy.yaml
dest: /etc/kubernetes/manifests/haproxy.yaml
period: 30
kubernetes_url: https://kubernetes.default:443
services:
default:
kubernetes:
server_opts: "check"
conf_parts:
frontend:
- mode tcp
- option tcpka
- bind *:6553
backend:
- mode tcp
- option tcpka
kube-system:
kubernetes-etcd:
server_opts: "check"
conf_parts:
frontend:
- mode tcp
- option tcpka
- bind *:2378
backend:
- mode tcp
- option tcpka
haproxy:
container_config_dir: /usr/local/etc/haproxy
host_config_dir: /etc/promenade/haproxy
period: 5
conf_parts:
global:
- maxconn 10240
defaults:
- timeout connect 5000ms
- timeout client 24h
- timeout server 24h
frontend: []
backend: []
images:
tags:
anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
haproxy: haproxy:1.8.3
pull_policy: "IfNotPresent"
pod:
lifecycle:
upgrades:
daemonsets:
pod_replacement_strategy: RollingUpdate
haproxy_anchor:
enabled: false
min_ready_seconds: 0
max_unavailable: 1
termination_grace_period:
haproxy_anchor:
timeout: 3600
resources:
enabled: false
haproxy_anchor:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "256Mi"
cpu: "2000m"
release_group: null

View File

@ -46,11 +46,11 @@ spec:
command:
- /opt/promenade/entrypoint.sh
- server
{{- if .Values.pod.env.promenade_api }}
{{- if $envAll.Values.pod.env.promenade_api }}
env:
{{- range .Values.pod.env.promenade_api }}
- name: {{ .name }}
value: {{ .value }}
{{- range $envAll.Values.pod.env.promenade_api }}
- name: {{ .name | quote }}
value: {{ .value | quote }}
{{- end }}
{{- end }}
ports:

View File

@ -20,7 +20,7 @@ Sample Document
tar_path: kubernetes/node/bin/kubelet
mode: 0555
images:
coredns: coredns/coredns:011
haproxy: haproxy:1.8.3
helm:
helm: lachlanevenson/k8s-helm:v2.7.2
kubernetes:
@ -91,13 +91,15 @@ Core Images
These images are used for essential functionality:
``coredns``
coredns_ is configured and used for Kubernetes API discovery during
``haproxy``
HAProxy_ is configured and used for Kubernetes API discovery during
bootstrapping.
``kubectl``
Used for label application and validation tasks during bootstrapping.
.. _HAProxy: https://www.haproxy.org/
Convenience Images
^^^^^^^^^^^^^^^^^^
@ -105,9 +107,6 @@ Convenience Images
The ``helm`` image is available for convenience.
.. _coredns: https://github.com/coredns/coredns
Packages
--------

View File

@ -30,12 +30,15 @@ Sample Document
- 8.8.4.4
kubernetes:
apiserver_port: 6443
haproxy_port: 6553
pod_cidr: 10.97.0.0/16
service_cidr: 10.96.0.0/16
service_ip: 10.96.0.1
etcd:
service_ip: 10.96.0.2
container_port: 2379
haproxy_port: 2378
hosts_entries:
- ip: 192.168.77.1
@ -72,6 +75,13 @@ Kubernetes
The ``kubernetes`` key contains:
``apiserver_port``
The port that the Kubernetes API server process will listen on on hosts where it runs.
``haproxy_port``
The port that HAProxy will listen on on each host. This port will be used
by the ``kubelet`` and ``kube-proxy`` to find API servers in the cluster.
``pod_cidr``
The CIDR from which the Kubernetes Controller Manager assigns pod IPs.

View File

@ -4,14 +4,25 @@ set -ex
PORT=${PORT:-9000}
UWSGI_TIMEOUT=${UWSGI_TIMEOUT:-300}
PROMENADE_THREADS=${PROMENADE_THREADS:-1}
PROMENADE_WORKERS=${PROMENADE_WORKERS:-4}
if [ "$1" = 'server' ]; then
exec uwsgi \
--http :${PORT} \
--http-timeout ${UWSGI_TIMEOUT} \
-z ${UWSGI_TIMEOUT} \
--http ":${PORT}" \
--http-timeout "${UWSGI_TIMEOUT}" \
--harakiri "${UWSGI_TIMEOUT}" \
--socket-timeout "${UWSGI_TIMEOUT}" \
--harakiri-verbose \
--lazy-apps \
--master \
--thunder-lock \
--die-on-term \
-z "${UWSGI_TIMEOUT}" \
--paste config:/etc/promenade/api-paste.ini \
--enable-threads -L \
--workers 4
--enable-threads \
--threads "${PROMENADE_THREADS}" \
--workers "${PROMENADE_WORKERS}"
fi
exec ${@}

View File

@ -30,7 +30,7 @@ data:
rotate 1
}
images:
coredns: coredns/coredns:0.9.9
haproxy: haproxy:1.8.3
helm:
helm: lachlanevenson/k8s-helm:v2.7.2
kubernetes:

View File

@ -21,12 +21,15 @@ data:
- 8.8.4.4
kubernetes:
apiserver_port: 6443
haproxy_port: 6553
pod_cidr: 10.97.0.0/16
service_cidr: 10.96.0.0/16
service_ip: 10.96.0.1
etcd:
service_ip: 10.96.0.2
container_port: 2379
haproxy_port: 2378
hosts_entries:
- ip: 192.168.77.1

View File

@ -69,6 +69,7 @@ metadata:
data:
description: Kubernetes components
chart_group:
- haproxy
- kubernetes-etcd
- kubernetes-apiserver
- kubernetes-controller-manager
@ -119,6 +120,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
substitutions:
-
src:
@ -160,7 +162,7 @@ data:
tags:
proxy: gcr.io/google_containers/hyperkube-amd64:v1.8.6
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
kubernetes_netloc: 127.0.0.1:6553
source:
type: local
location: /etc/genesis/armada/assets/charts
@ -175,6 +177,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
substitutions:
-
src:
@ -419,6 +422,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
substitutions:
-
src:
@ -482,28 +486,6 @@ metadata:
layeringDefinition:
abstract: false
layer: site
substitutions:
-
src:
schema: deckhand/CertificateAuthority/v1
name: kubernetes
path: .
dest:
path: '.values.tls.ca'
-
src:
schema: deckhand/Certificate/v1
name: coredns
path: .
dest:
path: '.values.tls.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: coredns
path: .
dest:
path: '.values.tls.key'
data:
chart_name: coredns
release: coredns
@ -514,39 +496,9 @@ data:
upgrade:
no_hooks: true
values:
coredns:
kubernetes_zones:
- cluster.local
- 10.96.0.0/16
- 10.97.0.0/16
upstream_nameservers:
- 8.8.8.8
- 8.8.4.4
zones:
- name: promenade
services:
- bind_name: apiserver.kubernetes
service:
name: kubernetes-apiserver
namespace: kube-system
- bind_name: etcd.kubernetes
service:
name: kubernetes-etcd
namespace: kube-system
- bind_name: etcd.calico
service:
name: calico-etcd
namespace: kube-system
images:
anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
coredns: coredns/coredns:0.9.9
tls:
ca: placeholder
cert: placeholder
key: placeholder
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
tags:
coredns: coredns/coredns:1.0.5
source:
type: local
location: /etc/genesis/armada/assets/charts
@ -555,6 +507,62 @@ data:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: haproxy
layeringDefinition:
abstract: false
layer: site
data:
chart_name: haproxy
release: haproxy
namespace: kube-system
timeout: 600
wait:
timeout: 600
upgrade:
no_hooks: true
values:
conf:
anchor:
kubernetes_url: https://kubernetes.default:443
services:
default:
kubernetes:
server_opts: "check"
conf_parts:
frontend:
- mode tcp
- option tcpka
- bind *:6553
backend:
- mode tcp
- option tcpka
kube-system:
kubernetes-etcd:
server_opts: "check"
conf_parts:
frontend:
- mode tcp
- option tcpka
- bind *:2378
backend:
- mode tcp
- option tcpka
images:
tags:
anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
haproxy: haproxy:1.8.3
source:
type: local
location: /etc/genesis/armada/assets/charts
subpath: haproxy
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: kubernetes-apiserver
@ -626,7 +634,7 @@ data:
values:
apiserver:
etcd:
endpoints: https://etcd.kubernetes.promenade:2379
endpoints: https://127.0.0.1:2378
images:
tags:
anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
@ -716,7 +724,7 @@ data:
cert: placeholder
key: placeholder
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
kubernetes_netloc: 127.0.0.1:6553
pod_cidr: 10.97.0.0/16
service_cidr: 10.96.0.0/16
@ -775,7 +783,7 @@ data:
key: placeholder
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
kubernetes_netloc: 127.0.0.1:6553
images:
tags:
@ -955,7 +963,7 @@ data:
no_hooks: true
values:
anchor:
etcdctl_endpoint: 10.96.0.2
etcdctl_endpoint: kubernetes-etcd.kube-system.svc.cluster.local
labels:
anchor:
node_selector_key: kubernetes-etcd
@ -1012,7 +1020,6 @@ data:
key: placeholder
service:
name: kubernetes-etcd
ip: 10.96.0.2
network:
service_client:
name: service_client
@ -1045,6 +1052,11 @@ data:
wait:
timeout: 600
values:
pod:
env:
promenade_api:
- name: PROMENADE_DEBUG
value: '1'
conf:
paste:
app:promenade-api:

View File

@ -31,7 +31,7 @@ data:
rotate 1
}
images:
coredns: coredns/coredns:0.9.9
haproxy: haproxy:1.8.3
helm:
helm: lachlanevenson/k8s-helm:v2.7.2
kubernetes:

View File

@ -20,12 +20,15 @@ data:
- 8.8.4.4
kubernetes:
apiserver_port: 6443
haproxy_port: 6553
pod_cidr: 10.97.0.0/16
service_cidr: 10.96.0.0/16
service_ip: 10.96.0.1
etcd:
service_ip: 10.96.0.2
container_port: 2379
haproxy_port: 2378
hosts_entries:
- ip: 192.168.77.1

View File

@ -86,6 +86,7 @@ metadata:
data:
description: Kubernetes components
chart_group:
- haproxy
- kubernetes-etcd
- kubernetes-apiserver
- kubernetes-controller-manager
@ -204,7 +205,7 @@ data:
tags:
proxy: gcr.io/google_containers/hyperkube-amd64:v1.8.6
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
kubernetes_netloc: 127.0.0.1:6553
source:
type: local
location: /etc/genesis/armada/assets/charts
@ -528,29 +529,6 @@ metadata:
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
substitutions:
-
src:
schema: deckhand/CertificateAuthority/v1
name: kubernetes
path: .
dest:
path: '.values.tls.ca'
-
src:
schema: deckhand/Certificate/v1
name: coredns
path: .
dest:
path: '.values.tls.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: coredns
path: .
dest:
path: '.values.tls.key'
data:
chart_name: coredns
release: coredns
@ -561,39 +539,9 @@ data:
upgrade:
no_hooks: true
values:
coredns:
kubernetes_zones:
- cluster.local
- 10.96.0.0/16
- 10.97.0.0/16
upstream_nameservers:
- 8.8.8.8
- 8.8.4.4
zones:
- name: promenade
services:
- bind_name: apiserver.kubernetes
service:
name: kubernetes-apiserver
namespace: kube-system
- bind_name: etcd.kubernetes
service:
name: kubernetes-etcd
namespace: kube-system
- bind_name: etcd.calico
service:
name: calico-etcd
namespace: kube-system
images:
anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
coredns: coredns/coredns:0.9.9
tls:
ca: placeholder
cert: placeholder
key: placeholder
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
tags:
coredns: coredns/coredns:1.0.5
source:
type: local
location: /etc/genesis/armada/assets/charts
@ -602,6 +550,62 @@ data:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: haproxy
layeringDefinition:
abstract: false
layer: site
data:
chart_name: haproxy
release: haproxy
namespace: kube-system
timeout: 600
wait:
timeout: 600
upgrade:
no_hooks: true
values:
conf:
anchor:
kubernetes_url: https://kubernetes.default:443
services:
default:
kubernetes:
server_opts: "check"
conf_parts:
frontend:
- mode tcp
- option tcpka
- bind *:6553
backend:
- mode tcp
- option tcpka
kube-system:
kubernetes-etcd:
server_opts: "check"
conf_parts:
frontend:
- mode tcp
- option tcpka
- bind *:2378
backend:
- mode tcp
- option tcpka
images:
tags:
anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
haproxy: haproxy:1.8.3
source:
type: local
location: /etc/genesis/armada/assets/charts
subpath: haproxy
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: kubernetes-apiserver
@ -673,7 +677,7 @@ data:
values:
apiserver:
etcd:
endpoints: https://etcd.kubernetes.promenade:2379
endpoints: https://127.0.0.1:2378
images:
tags:
anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
@ -763,7 +767,7 @@ data:
cert: placeholder
key: placeholder
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
kubernetes_netloc: 127.0.0.1:6553
pod_cidr: 10.97.0.0/16
service_cidr: 10.96.0.0/16
@ -822,7 +826,7 @@ data:
key: placeholder
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
kubernetes_netloc: 127.0.0.1:6553
images:
tags:
@ -1002,7 +1006,7 @@ data:
no_hooks: true
values:
anchor:
etcdctl_endpoint: 10.96.0.2
etcdctl_endpoint: kubernetes-etcd.kube-system.svc.cluster.local
labels:
anchor:
node_selector_key: kubernetes-etcd
@ -1059,7 +1063,6 @@ data:
key: placeholder
service:
name: kubernetes-etcd
ip: 10.96.0.2
network:
service_client:
name: service_client
@ -1756,6 +1759,11 @@ data:
wait:
timeout: 600
values:
pod:
env:
promenade_api:
- name: PROMENADE_DEBUG
value: '1'
images:
tags:
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1

View File

@ -33,7 +33,7 @@ class Generator:
ca='kubernetes',
cn='apiserver',
hosts=self._service_dns('kubernetes', 'default') +
['localhost', '127.0.0.1', 'apiserver.kubernetes.promenade'] +
['localhost', '127.0.0.1'] +
[self.config['KubernetesNetwork:kubernetes.service_ip']])
self.gen(
'certificate',
@ -75,25 +75,19 @@ class Generator:
cn='armada',
groups=['system:masters'])
# Certificates for coredns
self.gen('certificate', 'coredns', ca='kubernetes', cn='coredns')
# Certificates for Kubernetes's etcd servers
self.gen_etcd_certificates(
ca='kubernetes-etcd',
genesis=True,
service_name='kubernetes-etcd',
service_namespace='kube-system',
service_ip=self.config['KubernetesNetwork:etcd.service_ip'],
additional_hosts=['etcd.kubernetes.promenade'])
service_namespace='kube-system')
# Certificates for Calico's etcd servers
self.gen_etcd_certificates(
ca='calico-etcd',
service_name='calico-etcd',
service_namespace='kube-system',
service_ip=self.calico_etcd_service_ip,
additional_hosts=['etcd.calico.promenade'])
service_ip=self.calico_etcd_service_ip)
# Certificates for Calico node
self.gen(

View File

@ -71,8 +71,11 @@ data:
images:
type: object
properties:
# NOTE(mark-burnett): No longer used.
coredns:
$ref: '#/definitions/image'
haproxy:
$ref: '#/definitions/image'
helm:
type: object
properties:
@ -90,7 +93,7 @@ data:
- kubectl
additionalProperties: false
required:
- coredns
- haproxy
- helm
- kubernetes
additionalProperties: false

View File

@ -52,10 +52,16 @@ data:
etcd:
type: object
properties:
container_port:
type: integer
haproxy_port:
type: integer
# NOTE(mark-burnett): No longer used.
service_ip:
$ref: '#/definitions/ip_address'
required:
- service_ip
- container_port
- haproxy_port
additionalProperties: false
kubernetes:
@ -67,10 +73,16 @@ data:
$ref: '#/definitions/ip_address'
service_cidr:
$ref: '#/definitions/cidr'
apiserver_port:
type: integer
haproxy_port:
type: integer
required:
- pod_cidr
- service_cidr
- service_ip
- apiserver_port
- haproxy_port
additionalProperties: false
hosts_entries:
type: array

View File

@ -1,13 +0,0 @@
promenade {
file /etc/coredns/zones/promenade
errors stdout
log stdout
}
. {
{%- if config['KubernetesNetwork:dns.upstream_servers'] is defined %}
proxy . {%- for server in config['KubernetesNetwork:dns.upstream_servers'] %} {{ server -}}{%- endfor %}
{%- endif %}
errors stdout
log stdout
}

View File

@ -1,11 +0,0 @@
$ORIGIN promenade.
promenade. IN SOA @ root {{ now }} 3h 15m 1w 1d
{%- if config['KubernetesNode:join_ip'] is defined %}
apiserver.kubernetes IN A {{ config['KubernetesNode:join_ip'] }}
{%- else %}
apiserver.kubernetes IN A 127.0.0.1
{%- endif %}
etcd.kubernetes IN A 127.0.0.1

View File

@ -2,7 +2,7 @@
apiVersion: v1
clusters:
- cluster:
server: https://apiserver.kubernetes.promenade:6443
server: https://127.0.0.1:6553
certificate-authority: pki/cluster-ca.pem
name: kubernetes
contexts:

View File

@ -2,7 +2,7 @@
apiVersion: v1
clusters:
- cluster:
server: https://apiserver.kubernetes.promenade:6443
server: https://127.0.0.1:6553
certificate-authority: pki/cluster-ca.pem
name: kubernetes
contexts:

View File

@ -1,30 +0,0 @@
---
apiVersion: v1
kind: Pod
metadata:
name: coredns
namespace: kube-system
labels:
# NOTE(mark-burnett): This is a host initialization Pod and should not be
# included in the Kubernetes DNS service, so ensure it does not get
# selected.
coredns-service: disabled
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
hostNetwork: true
containers:
- name: coredns
image: {{ config['HostSystem:images.coredns'] }}
command:
- /coredns
- -conf
- /etc/coredns/Corefile
volumeMounts:
- name: config
mountPath: /etc/coredns
readOnly: true
volumes:
- name: config
hostPath:
path: /etc/coredns

View File

@ -0,0 +1,39 @@
---
apiVersion: v1
kind: Pod
metadata:
name: haproxy
namespace: kube-system
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
hostNetwork: true
containers:
- name: haproxy
image: {{ config['HostSystem:images.haproxy'] }}
imagePullPolicy: IfNotPresent
hostNetwork: true
env:
- name: HAPROXY_CONF
value: /usr/local/etc/haproxy/haproxy.cfg
command:
- /bin/sh
- -c
- |
set -eux
while [ ! -s "$HAPROXY_CONF" ]; do
echo Waiting for "HAPROXY_CONF"
sleep 1
done
haproxy -f "$HAPROXY_CONF"
volumeMounts:
- name: etc
mountPath: /usr/local/etc/haproxy
readOnly: true
volumes:
- name: etc
hostPath:
path: /etc/promenade/haproxy

View File

@ -0,0 +1,43 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
global
maxconn 10240
defaults
mode tcp
option tcpka
timeout connect 5000ms
timeout client 24h
timeout server 24h
frontend default-kubernetes-fe
bind *:{{ config['KubernetesNetwork:kubernetes.haproxy_port'] }}
default_backend default-kubernetes-be
{% set ip = config.get_first('KubernetesNode:join_ip', 'Genesis:ip') -%}
backend default-kubernetes-be
option tcp-check
{%- set port = config['KubernetesNetwork:kubernetes.apiserver_port'] %}
server s{{ ip }} {{ ip }}:{{ port }} check port {{ port }}
frontend kube-system-kubernetes-etcd-fe
bind *:{{ config['KubernetesNetwork:etcd.haproxy_port'] }}
default_backend kube-system-kubernetes-etcd-be
backend kube-system-kubernetes-etcd-be
option tcp-check
{%- set port = config['KubernetesNetwork:etcd.container_port'] %}
server s{{ ip }} {{ ip }}:{{ port }} check port {{ port }}

View File

@ -1,6 +1,6 @@
options timeout:1 attempts:1
nameserver 127.0.0.1
nameserver 10.96.0.10
{% for server in config['KubernetesNetwork:dns.upstream_servers'] | default([]) %}
nameserver {{ server }}
{%- endfor %}

View File

@ -2,7 +2,7 @@
apiVersion: v1
clusters:
- cluster:
server: https://apiserver.kubernetes.promenade:6443
server: https://127.0.0.1:6553
certificate-authority: pki/cluster-ca.pem
name: kubernetes
contexts:

View File

@ -2,7 +2,7 @@
apiVersion: v1
clusters:
- cluster:
server: https://apiserver.kubernetes.promenade:6443
server: https://127.0.0.1:6553
certificate-authority: pki/cluster-ca.pem
name: kubernetes
contexts:

View File

@ -7,7 +7,7 @@ registry_down() {
}
registry_list_images() {
FILES=($(echo "$(config_configuration)" | xargs -n 1 -I DIRNAME find DIRNAME -type f -name '*.yaml'))
FILES=($(config_configuration | xargs -n 1 -I DIRNAME find DIRNAME -type f -name '*.yaml'))
HOSTNAME_REGEX='[a-zA-Z0-9][a-zA-Z0-9_-]{0,62}'
DOMAIN_NAME_REGEX="${HOSTNAME_REGEX}(\.${HOSTNAME_REGEX})*"

View File

@ -12,6 +12,8 @@ validate_etcd_membership() {
shift 2
EXPECTED_MEMBERS="${*}"
# NOTE(mark-burnett): Wait a moment for disks in test environment to settle.
sleep 10
log Validating "${CLUSTER}" etcd membership via "${VM}"
FOUND_MEMBERS=$(etcdctl_member_list "${CLUSTER}" "${VM}" | tr '\n' ' ' | sed 's/ $//')

View File

@ -129,7 +129,7 @@ vm_create() {
--cpu host \
--graphics vnc,listen=0.0.0.0 \
--noautoconsole \
--network network=promenade \
--network "network=promenade,model=virtio" \
--vcpus "$(config_vm_vcpus)" \
--memory "$(config_vm_memory)" \
--import \

View File

@ -82,6 +82,7 @@ metadata:
data:
description: Kubernetes components
chart_group:
- haproxy
- kubernetes-etcd
- kubernetes-apiserver
- kubernetes-controller-manager
@ -123,23 +124,23 @@ metadata:
src:
schema: deckhand/CertificateAuthority/v1
name: kubernetes
path: $
path: .
dest:
path: '$.values.secrets.tls.ca'
path: '.values.secrets.tls.ca'
-
src:
schema: deckhand/Certificate/v1
name: proxy
path: $
path: .
dest:
path: '$.values.secrets.tls.cert'
path: '.values.secrets.tls.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: proxy
path: $
path: .
dest:
path: '$.values.secrets.tls.key'
path: '.values.secrets.tls.key'
data:
chart_name: proxy
release: kubernetes-proxy
@ -157,7 +158,7 @@ data:
tags:
proxy: ${IMAGE_HYPERKUBE}
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
kubernetes_netloc: 127.0.0.1:6553
source:
type: local
location: /etc/genesis/armada/assets/charts
@ -178,147 +179,147 @@ metadata:
src:
schema: deckhand/CertificateAuthority/v1
name: calico-etcd
path: $
path: .
dest:
path: '$.values.secrets.tls.client.ca'
path: '.values.secrets.tls.client.ca'
-
src:
schema: deckhand/CertificateAuthority/v1
name: calico-etcd-peer
path: $
path: .
dest:
path: '$.values.secrets.tls.peer.ca'
path: '.values.secrets.tls.peer.ca'
-
src:
schema: deckhand/Certificate/v1
name: calico-etcd-anchor
path: $
path: .
dest:
path: '$.values.secrets.anchor.tls.cert'
path: '.values.secrets.anchor.tls.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-anchor
path: $
path: .
dest:
path: '$.values.secrets.anchor.tls.key'
path: '.values.secrets.anchor.tls.key'
-
src:
schema: deckhand/Certificate/v1
name: calico-etcd-${GENESIS_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[0].tls.client.cert'
path: '.values.nodes[0].tls.client.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-${GENESIS_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[0].tls.client.key'
path: '.values.nodes[0].tls.client.key'
-
src:
schema: deckhand/Certificate/v1
name: calico-etcd-${GENESIS_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[0].tls.peer.cert'
path: '.values.nodes[0].tls.peer.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-${GENESIS_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[0].tls.peer.key'
path: '.values.nodes[0].tls.peer.key'
-
src:
schema: deckhand/Certificate/v1
name: calico-etcd-${MASTER1_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[1].tls.client.cert'
path: '.values.nodes[1].tls.client.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-${MASTER1_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[1].tls.client.key'
path: '.values.nodes[1].tls.client.key'
-
src:
schema: deckhand/Certificate/v1
name: calico-etcd-${MASTER1_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[1].tls.peer.cert'
path: '.values.nodes[1].tls.peer.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-${MASTER1_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[1].tls.peer.key'
path: '.values.nodes[1].tls.peer.key'
-
src:
schema: deckhand/Certificate/v1
name: calico-etcd-${MASTER2_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[2].tls.client.cert'
path: '.values.nodes[2].tls.client.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-${MASTER2_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[2].tls.client.key'
path: '.values.nodes[2].tls.client.key'
-
src:
schema: deckhand/Certificate/v1
name: calico-etcd-${MASTER2_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[2].tls.peer.cert'
path: '.values.nodes[2].tls.peer.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-${MASTER2_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[2].tls.peer.key'
path: '.values.nodes[2].tls.peer.key'
-
src:
schema: deckhand/Certificate/v1
name: calico-etcd-${WORKER_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[3].tls.client.cert'
path: '.values.nodes[3].tls.client.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-${WORKER_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[3].tls.client.key'
path: '.values.nodes[3].tls.client.key'
-
src:
schema: deckhand/Certificate/v1
name: calico-etcd-${WORKER_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[3].tls.peer.cert'
path: '.values.nodes[3].tls.peer.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-${WORKER_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[3].tls.peer.key'
path: '.values.nodes[3].tls.peer.key'
data:
chart_name: etcd
@ -421,23 +422,23 @@ metadata:
src:
schema: deckhand/CertificateAuthority/v1
name: calico-etcd
path: $
path: .
dest:
path: '$.values.etcd.tls.ca'
path: '.values.etcd.tls.ca'
-
src:
schema: deckhand/Certificate/v1
name: calico-node
path: $
path: .
dest:
path: '$.values.etcd.tls.cert'
path: '.values.etcd.tls.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: calico-node
path: $
path: .
dest:
path: '$.values.etcd.tls.key'
path: '.values.etcd.tls.key'
data:
chart_name: calico
release: calico
@ -479,28 +480,6 @@ metadata:
abstract: false
layer: site
storagePolicy: cleartext
substitutions:
-
src:
schema: deckhand/CertificateAuthority/v1
name: kubernetes
path: $
dest:
path: '$.values.tls.ca'
-
src:
schema: deckhand/Certificate/v1
name: coredns
path: $
dest:
path: '$.values.tls.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: coredns
path: $
dest:
path: '$.values.tls.key'
data:
chart_name: coredns
release: coredns
@ -509,39 +488,9 @@ data:
upgrade:
no_hooks: true
values:
coredns:
kubernetes_zones:
- cluster.local
- 10.96.0.0/16
- 10.97.0.0/16
upstream_nameservers:
- 8.8.8.8
- 8.8.4.4
zones:
- name: promenade
services:
- bind_name: apiserver.kubernetes
service:
name: kubernetes-apiserver
namespace: kube-system
- bind_name: etcd.kubernetes
service:
name: kubernetes-etcd
namespace: kube-system
- bind_name: etcd.calico
service:
name: calico-etcd
namespace: kube-system
images:
anchor: ${IMAGE_HYPERKUBE}
coredns: ${IMAGE_COREDNS}
tls:
ca: placeholder
cert: placeholder
key: placeholder
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
tags:
coredns: ${IMAGE_COREDNS}
source:
type: local
location: /etc/genesis/armada/assets/charts
@ -550,6 +499,62 @@ data:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: haproxy
layeringDefinition:
abstract: false
layer: site
data:
chart_name: haproxy
release: haproxy
namespace: kube-system
timeout: 600
wait:
timeout: 600
upgrade:
no_hooks: true
values:
conf:
anchor:
kubernetes_url: https://kubernetes.default:443
services:
default:
kubernetes:
server_opts: "check"
conf_parts:
frontend:
- mode tcp
- option tcpka
- bind *:6553
backend:
- mode tcp
- option tcpka
kube-system:
kubernetes-etcd:
server_opts: "check"
conf_parts:
frontend:
- mode tcp
- option tcpka
- bind *:2378
backend:
- mode tcp
- option tcpka
images:
tags:
anchor: ${IMAGE_HYPERKUBE}
haproxy: ${IMAGE_HAPROXY}
source:
type: local
location: /etc/genesis/armada/assets/charts
subpath: haproxy
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: kubernetes-apiserver
@ -562,52 +567,52 @@ metadata:
src:
schema: deckhand/CertificateAuthority/v1
name: kubernetes
path: $
path: .
dest:
path: $.values.secrets.tls.ca
path: .values.secrets.tls.ca
-
src:
schema: deckhand/Certificate/v1
name: apiserver
path: $
path: .
dest:
path: $.values.secrets.tls.cert
path: .values.secrets.tls.cert
-
src:
schema: deckhand/CertificateKey/v1
name: apiserver
path: $
path: .
dest:
path: $.values.secrets.tls.key
path: .values.secrets.tls.key
-
src:
schema: deckhand/CertificateAuthority/v1
name: kubernetes-etcd
path: $
path: .
dest:
path: $.values.secrets.etcd.tls.ca
path: .values.secrets.etcd.tls.ca
-
src:
schema: deckhand/Certificate/v1
name: apiserver-etcd
path: $
path: .
dest:
path: $.values.secrets.etcd.tls.cert
path: .values.secrets.etcd.tls.cert
-
src:
schema: deckhand/CertificateKey/v1
name: apiserver-etcd
path: $
path: .
dest:
path: $.values.secrets.etcd.tls.key
path: .values.secrets.etcd.tls.key
-
src:
schema: deckhand/PublicKey/v1
name: service-account
path: $
path: .
dest:
path: $.values.secrets.service_account.public_key
path: .values.secrets.service_account.public_key
data:
chart_name: apiserver
@ -619,7 +624,7 @@ data:
values:
apiserver:
etcd:
endpoints: https://etcd.kubernetes.promenade:2379
endpoints: https://127.0.0.1:2378
images:
tags:
anchor: ${IMAGE_HYPERKUBE}
@ -661,31 +666,31 @@ metadata:
src:
schema: deckhand/CertificateAuthority/v1
name: kubernetes
path: $
path: .
dest:
path: $.values.secrets.tls.ca
path: .values.secrets.tls.ca
-
src:
schema: deckhand/Certificate/v1
name: controller-manager
path: $
path: .
dest:
path: $.values.secrets.tls.cert
path: .values.secrets.tls.cert
-
src:
schema: deckhand/CertificateKey/v1
name: controller-manager
path: $
path: .
dest:
path: $.values.secrets.tls.key
path: .values.secrets.tls.key
-
src:
schema: deckhand/PrivateKey/v1
name: service-account
path: $
path: .
dest:
path: $.values.secrets.service_account.private_key
path: .values.secrets.service_account.private_key
data:
chart_name: controller_manager
@ -707,7 +712,7 @@ data:
cert: placeholder
key: placeholder
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
kubernetes_netloc: 127.0.0.1:6553
pod_cidr: 10.97.0.0/16
service_cidr: 10.96.0.0/16
@ -731,23 +736,23 @@ metadata:
src:
schema: deckhand/CertificateAuthority/v1
name: kubernetes
path: $
path: .
dest:
path: $.values.secrets.tls.ca
path: .values.secrets.tls.ca
-
src:
schema: deckhand/Certificate/v1
name: scheduler
path: $
path: .
dest:
path: $.values.secrets.tls.cert
path: .values.secrets.tls.cert
-
src:
schema: deckhand/CertificateKey/v1
name: scheduler
path: $
path: .
dest:
path: $.values.secrets.tls.key
path: .values.secrets.tls.key
data:
chart_name: scheduler
@ -764,7 +769,7 @@ data:
key: placeholder
network:
kubernetes_netloc: apiserver.kubernetes.promenade:6443
kubernetes_netloc: 127.0.0.1:6553
images:
tags:
@ -791,118 +796,118 @@ metadata:
src:
schema: deckhand/CertificateAuthority/v1
name: kubernetes-etcd
path: $
path: .
dest:
path: '$.values.secrets.tls.client.ca'
path: '.values.secrets.tls.client.ca'
-
src:
schema: deckhand/CertificateAuthority/v1
name: kubernetes-etcd-peer
path: $
path: .
dest:
path: '$.values.secrets.tls.peer.ca'
path: '.values.secrets.tls.peer.ca'
-
src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-anchor
path: $
path: .
dest:
path: '$.values.secrets.anchor.tls.cert'
path: '.values.secrets.anchor.tls.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-anchor
path: $
path: .
dest:
path: '$.values.secrets.anchor.tls.key'
path: '.values.secrets.anchor.tls.key'
-
src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-${GENESIS_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[0].tls.client.cert'
path: '.values.nodes[0].tls.client.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-${GENESIS_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[0].tls.client.key'
path: '.values.nodes[0].tls.client.key'
-
src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-${GENESIS_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[0].tls.peer.cert'
path: '.values.nodes[0].tls.peer.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-${GENESIS_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[0].tls.peer.key'
path: '.values.nodes[0].tls.peer.key'
-
src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-${MASTER1_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[1].tls.client.cert'
path: '.values.nodes[1].tls.client.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-${MASTER1_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[1].tls.client.key'
path: '.values.nodes[1].tls.client.key'
-
src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-${MASTER1_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[1].tls.peer.cert'
path: '.values.nodes[1].tls.peer.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-${MASTER1_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[1].tls.peer.key'
path: '.values.nodes[1].tls.peer.key'
-
src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-${MASTER2_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[2].tls.client.cert'
path: '.values.nodes[2].tls.client.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-${MASTER2_HOSTNAME}
path: $
path: .
dest:
path: '$.values.nodes[2].tls.client.key'
path: '.values.nodes[2].tls.client.key'
-
src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-${MASTER2_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[2].tls.peer.cert'
path: '.values.nodes[2].tls.peer.cert'
-
src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-${MASTER2_HOSTNAME}-peer
path: $
path: .
dest:
path: '$.values.nodes[2].tls.peer.key'
path: '.values.nodes[2].tls.peer.key'
data:
chart_name: etcd
@ -913,7 +918,7 @@ data:
no_hooks: true
values:
anchor:
etcdctl_endpoint: 10.96.0.2
etcdctl_endpoint: kubernetes-etcd
labels:
anchor:
node_selector_key: kubernetes-etcd
@ -962,7 +967,6 @@ data:
key: placeholder
service:
name: kubernetes-etcd
ip: 10.96.0.2
network:
service_client:
name: service_client

View File

@ -21,12 +21,15 @@ data:
- 8.8.4.4
kubernetes:
apiserver_port: 6443
haproxy_port: 6553
pod_cidr: 10.97.0.0/16
service_cidr: 10.96.0.0/16
service_ip: 10.96.0.1
etcd:
service_ip: 10.96.0.2
container_port: 2379
haproxy_port: 2378
---
schema: promenade/Docker/v1
metadata:
@ -60,7 +63,7 @@ data:
tar_path: kubernetes/node/bin/kubelet
mode: 0555
images:
coredns: ${IMAGE_COREDNS}
haproxy: ${IMAGE_HAPROXY}
helm:
helm: ${IMAGE_HELM}
kubernetes:

View File

@ -3,9 +3,10 @@ IMAGE_CALICO_CNI=quay.io/calico/cni:v1.11.2
IMAGE_CALICO_CTL=quay.io/calico/ctl:v1.6.3
IMAGE_CALICO_KUBE_CONTROLLERS=quay.io/calico/kube-controllers:v1.0.2
IMAGE_CALICO_NODE=quay.io/calico/node:v2.6.5
IMAGE_COREDNS=coredns/coredns:0.9.9
IMAGE_COREDNS=coredns/coredns:1.0.5
IMAGE_DEP_CHECK=quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
IMAGE_ETCD=quay.io/coreos/etcd:v3.2.14
IMAGE_HAPROXY=haproxy:1.8.3
IMAGE_HELM=lachlanevenson/k8s-helm:v2.7.2
IMAGE_HYPERKUBE=gcr.io/google_containers/hyperkube-amd64:v1.8.6
IMAGE_TILLER=gcr.io/kubernetes-helm/tiller:v2.7.2