diff --git a/charts/coredns/templates/bin/_anchor.tpl b/charts/coredns/templates/bin/_anchor.tpl deleted file mode 100644 index c8187967..00000000 --- a/charts/coredns/templates/bin/_anchor.tpl +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/sh - -{{- $envAll := . }} - -set -x - -export MANIFEST_PATH=/host{{ .Values.anchor.kubelet.manifest_path }}/{{ .Values.service.name }}.yaml -export ETC_PATH=/host{{ .Values.coredns.host_etc_path }} -TOKEN_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token -CA_CERT_PATH=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - -copy_etc_files() { - mkdir -p $ETC_PATH/zones - cp /configmap/* /secret/* $ETC_PATH - create_corefile -} - -create_corefile() { - cat < $ETC_PATH/Corefile -promenade { - file /etc/coredns/zones/promenade - loadbalance - errors stdout - log stdout -} - -. { - kubernetes{{- range .Values.coredns.kubernetes_zones }} {{ . -}}{{- end }} { - endpoint https://{{ .Values.network.kubernetes_netloc }} - tls /etc/coredns/coredns.pem /etc/coredns/coredns-key.pem /etc/coredns/cluster-ca.pem - - pods insecure - } - {{- if .Values.coredns.upstream_nameservers }} - {{ range .Values.coredns.upstream_nameservers }} - proxy . {{ . }} - {{- end }} - {{- end }} - - loadbalance - cache {{ .Values.coredns.cache.ttl }} - - errors stdout - log stdout -} -EOCOREFILE -} - -create_manifest() { - mkdir -p $(dirname $MANIFEST_PATH) -# XXX liveness/readiness probes - cat < $MANIFEST_PATH ---- -apiVersion: v1 -kind: Pod -metadata: - name: {{ .Values.service.name }} - namespace: {{ .Release.Namespace }} - labels: - {{ .Values.service.name }}-service: enabled - anchor-managed: enabled -spec: - hostNetwork: true - containers: - - name: coredns - image: {{ .Values.images.coredns }} - command: - - /coredns - - -conf - - /etc/coredns/Corefile - volumeMounts: - - name: etc - mountPath: /etc/coredns - volumes: - - name: etc - hostPath: - path: {{ .Values.coredns.host_etc_path }} -EODOC -} - -update_managed_zones() { -{{- range .Values.coredns.zones }} - -FILENAME="$ETC_PATH/zones/{{ .name }}" -NEXT_FILENAME="${FILENAME}-next" -SUCCESS=1 -NOW=$(date +%s) - -# Add Header -cat < $NEXT_FILENAME -\$ORIGIN {{ .name }}. -{{ .name }}. IN SOA @ root $NOW 3h 15m 1w 1d - -EOBIND -{{ range .services }} -# Don't accidentally log service account token -set +x -SERVICE_IPS=$(kubectl \ - --server https://{{ $envAll.Values.network.kubernetes_netloc }} \ - --certificate-authority $CA_CERT_PATH \ - --token $(cat $TOKEN_PATH) \ - -n {{ .service.namespace }} \ - get ep {{ .service.name }} \ - -o 'jsonpath={.subsets[*].addresses[*].ip}') -set -x -if [ "x$SERVICE_IPS" != "x" ]; then - for IP in $SERVICE_IPS; do - echo {{ .bind_name }} IN A $IP >> $NEXT_FILENAME - done -else - echo Failed to upate zone file for {{ .name }} - SUCCESS=0 -fi -{{- end }} - -if [ $SUCCESS = 1 ]; then - echo Replacing zone file $FILENAME - mv $NEXT_FILENAME $FILENAME -fi -{{- end }} -} - -copy_etc_files -create_manifest - -while true; do - update_managed_zones - - sleep {{ .Values.anchor.period }} -done diff --git a/charts/coredns/templates/configmap-bin.yaml b/charts/coredns/templates/configmap-bin.yaml deleted file mode 100644 index 3b76ec58..00000000 --- a/charts/coredns/templates/configmap-bin.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Values.service.name }}-bin -data: - anchor: |+ -{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/coredns/templates/configmap-etc.yaml b/charts/coredns/templates/configmap-etc.yaml index 54714212..b7a09beb 100644 --- a/charts/coredns/templates/configmap-etc.yaml +++ b/charts/coredns/templates/configmap-etc.yaml @@ -4,5 +4,4 @@ kind: ConfigMap metadata: name: {{ .Values.service.name }}-etc data: - cluster-ca.pem: {{ .Values.tls.ca | quote }} - coredns.pem: {{ .Values.tls.cert | quote }} + Corefile: {{ .Values.conf.coredns.corefile | quote }} diff --git a/charts/coredns/templates/daemonset.yaml b/charts/coredns/templates/daemonset.yaml deleted file mode 100644 index 95825e84..00000000 --- a/charts/coredns/templates/daemonset.yaml +++ /dev/null @@ -1,83 +0,0 @@ ---- -apiVersion: "extensions/v1beta1" -kind: DaemonSet -metadata: - name: {{ .Values.service.name }}-anchor - labels: - application: coredns - component: coredns-anchor -spec: - selector: - matchLabels: - {{ .Values.service.name | quote }}: anchor - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - {{ .Values.service.name | quote }}: anchor - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} - configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} - spec: - hostNetwork: true - {{- if .Values.node_selector.key }} - nodeSelector: - {{ .Values.node_selector.key }}: {{ .Values.node_selector.value }} - {{- end }} - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - - key: CriticalAddonsOnly - operator: Exists - serviceAccountName: {{ .Values.service.name }} - containers: - - name: anchor - image: {{ .Values.images.anchor }} - command: - - /tmp/bin/anchor -# livenessProbe: -# httpGet: -# path: /healthz -# port: 10249 -# initialDelaySeconds: 15 -# periodSeconds: 15 -# failureThreshold: 3 -# readinessProbe: -# exec: -# command: -# - sh -# - -c -# - |- -# set -ex -# -# iptables-save | grep 'default/kubernetes:https' -# initialDelaySeconds: 15 -# periodSeconds: 15 - volumeMounts: - - name: bin - mountPath: /tmp/bin - - name: etc - mountPath: /configmap - - name: host - mountPath: /host - - name: secret - mountPath: /secret - volumes: - - name: bin - configMap: - name: {{ .Values.service.name }}-bin - defaultMode: 0555 - - name: etc - configMap: - name: {{ .Values.service.name }}-etc - defaultMode: 0444 - - name: host - hostPath: - path: / - - name: secret - secret: - secretName: {{ .Values.service.name }} - defaultMode: 0444 diff --git a/charts/coredns/templates/deployment.yaml b/charts/coredns/templates/deployment.yaml new file mode 100644 index 00000000..94d3a760 --- /dev/null +++ b/charts/coredns/templates/deployment.yaml @@ -0,0 +1,85 @@ +{{/* +Copyright 2018 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: coredns + labels: + {{ .Values.service.name }}: enabled + kubernetes.io/name: "CoreDNS" +spec: + replicas: 2 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + {{ .Values.service.name }}: enabled + template: + metadata: + labels: + {{ .Values.service.name }}: enabled + spec: + serviceAccountName: coredns + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: {{ .Values.service.name }} + operator: In + values: + - enabled + topologyKey: kubernetes.io/hostname + containers: + - name: coredns + image: {{ .Values.images.tags.coredns | quote }} + imagePullPolicy: {{ .Values.images.pull_policy | quote }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: {{ .Values.service.name }}-etc + items: + - key: Corefile + path: Corefile diff --git a/charts/coredns/templates/rbac.yaml b/charts/coredns/templates/rbac.yaml index 7bc65531..6d95c858 100644 --- a/charts/coredns/templates/rbac.yaml +++ b/charts/coredns/templates/rbac.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: coredns - namespace: kube-system + namespace: {{ .Release.Namespace }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -17,7 +17,7 @@ subjects: apiGroup: rbac.authorization.k8s.io - kind: ServiceAccount name: coredns - namespace: kube-system + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: system:coredns diff --git a/charts/coredns/templates/secret.yaml b/charts/coredns/templates/secret.yaml deleted file mode 100644 index 1f6d585f..00000000 --- a/charts/coredns/templates/secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Values.service.name }} -type: Opaque -data: - coredns-key.pem: {{ .Values.tls.key | b64enc }} diff --git a/charts/coredns/templates/service.yaml b/charts/coredns/templates/service.yaml index 621b8140..6d2e3bea 100644 --- a/charts/coredns/templates/service.yaml +++ b/charts/coredns/templates/service.yaml @@ -3,9 +3,12 @@ apiVersion: v1 kind: Service metadata: name: {{ .Values.service.name }} + labels: + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" spec: selector: - {{ .Values.service.name }}-service: enabled + {{ .Values.service.name }}: enabled clusterIP: {{ .Values.service.ip }} ports: - name: dns diff --git a/charts/coredns/values.yaml b/charts/coredns/values.yaml index 96def6ee..4b50ceda 100644 --- a/charts/coredns/values.yaml +++ b/charts/coredns/values.yaml @@ -1,38 +1,26 @@ -anchor: - kubelet: - manifest_path: /etc/kubernetes/manifests - node_selector: {} - period: 30 - -tls: - ca: placeholder - cert: placeholder - key: placeholder - -coredns: - kubernetes_zones: - - cluster.local - cache: - ttl: 60 - host_etc_path: /etc/coredns - upstream_nameservers: - - 8.8.8.8 - - 8.8.4.4 - - zones: - - name: promenade - services: - - bind_name: apiserver.kubernetes - service: - name: kubernetes - namespace: default +conf: + coredns: + corefile: | + .:53 { + errors + health + autopath @kubernetes + kubernetes cluster.local 10.96.0.0/16 10.97.0.0/16 { + pods insecure + fallthrough in-addr.arpa ip6.arpa + upstream 8.8.8.8 + upstream 8.8.4.4 + } + prometheus :9153 + proxy . 8.8.8.8 + proxy . 8.8.4.4 + cache 30 + } images: - anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6 - coredns: coredns/coredns:0.9.9 - -network: - kubernetes_netloc: 10.96.0.1 + tags: + coredns: coredns/coredns:1.0.5 + pull_policy: "IfNotPresent" node_selector: {} diff --git a/charts/haproxy/Chart.yaml b/charts/haproxy/Chart.yaml new file mode 100644 index 00000000..d2702589 --- /dev/null +++ b/charts/haproxy/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A chart for using HAProxy for Kubernetes API server discovery +name: haproxy +version: 0.1.0 diff --git a/charts/haproxy/requirements.yaml b/charts/haproxy/requirements.yaml new file mode 100644 index 00000000..4b156320 --- /dev/null +++ b/charts/haproxy/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/charts/haproxy/templates/bin/_anchor.tpl b/charts/haproxy/templates/bin/_anchor.tpl new file mode 100644 index 00000000..237adbc3 --- /dev/null +++ b/charts/haproxy/templates/bin/_anchor.tpl @@ -0,0 +1,129 @@ +#!/bin/sh +{{/* +Copyright 2018 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $envAll := . }} + +set -x + +compare_copy_files() { + {{- range .Values.conf.anchor.files_to_copy }} + if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then + mkdir -p $(dirname /host{{ .dest }}) + cp {{ .source }} /host{{ .dest }} + fi + {{- end }} +} + +install_config() { + SUCCESS=1 + # Inject global and default config + mkdir -p $(dirname "$HAPROXY_CONF") + cp "$HAPROXY_HEADER" "$NEXT_HAPROXY_CONF" + + {{- range $namespace, $services := $envAll.Values.conf.anchor.services }} + {{- range $service, $svc_data := $services }} + echo Constructing config for namespace=\"{{ $namespace }}\" service=\"{{ $service }}\" + + # NOTE(mark-burnett): Don't accidentally log service account token. + set +x + SERVICE_IPS=$(kubectl \ + --server "$KUBE_URL" \ + --certificate-authority "$KUBE_CA" \ + --token $(cat "$KUBE_TOKEN") \ + --namespace {{ $namespace }} \ + get endpoints {{ $service }} \ + -o 'jsonpath={.subsets[0].addresses[*].ip}') + DEST_PORT=$(kubectl \ + --server "$KUBE_URL" \ + --certificate-authority "$KUBE_CA" \ + --token $(cat "$KUBE_TOKEN") \ + --namespace {{ $namespace }} \ + get endpoints {{ $service }} \ + -o 'jsonpath={.subsets[0].ports[0].port}') + set -x + + if [ "x$SERVICE_IPS" != "x" ]; then + if [ "x$DEST_PORT" != "x" ]; then + IDENTIFIER=$(echo "{{ $namespace }}-{{ $service }}") + # Add frontend config + echo >> "$NEXT_HAPROXY_CONF" + echo "frontend ${IDENTIFIER}-fe" >> "$NEXT_HAPROXY_CONF" + {{- range $envAll.Values.conf.haproxy.conf_parts.frontend }} + echo " {{ . }}" >> "$NEXT_HAPROXY_CONF" + {{- end }} + {{- range $svc_data.conf_parts.frontend }} + echo " {{ . }}" >> "$NEXT_HAPROXY_CONF" + {{- end }} + echo " default_backend ${IDENTIFIER}-be" >> "$NEXT_HAPROXY_CONF" + + # Add backend config + echo >> "$NEXT_HAPROXY_CONF" + echo "backend ${IDENTIFIER}-be" >> "$NEXT_HAPROXY_CONF" + {{- range $envAll.Values.conf.haproxy.conf_parts.backend }} + echo " {{ . }}" >> "$NEXT_HAPROXY_CONF" + {{- end }} + {{- range $svc_data.conf_parts.backend }} + echo " {{ . }}" >> "$NEXT_HAPROXY_CONF" + {{- end }} + + for IP in $SERVICE_IPS; do + echo " server s$IP $IP:$DEST_PORT" {{ $svc_data.server_opts | quote }} >> "$NEXT_HAPROXY_CONF" + done + else + echo Failed to get destination port for service. + SUCCESS=0 + fi + else + echo Failed to get endpoint IPs for service. + SUCCESS=0 + fi + {{- end }} + {{- end }} + + if [ $SUCCESS = 1 ]; then + mkdir -p $(dirname "$HAPROXY_CONF") + if ! cmp -s "$HAPROXY_CONF" "$NEXT_HAPROXY_CONF"; then + echo Replacing HAProxy config file "$HAPROXY_CONF" with: + cat "$NEXT_HAPROXY_CONF" + echo + mv "$NEXT_HAPROXY_CONF" "$HAPROXY_CONF" + else + echo HAProxy config file unchanged. + fi + fi +} + +cleanup() { + {{- range .Values.conf.anchor.files_to_copy }} + rm -f /host{{ .dest }} + {{- end }} + rm -f "$HAPROXY_CONF" "$NEXT_HAPROXY_CONF" +} + +while true; do + if [ -e /tmp/stop ]; then + echo Stopping + cleanup + break + fi + + install_config + + compare_copy_files + + sleep {{ .Values.conf.anchor.period }} +done diff --git a/charts/haproxy/templates/bin/_pre_stop.tpl b/charts/haproxy/templates/bin/_pre_stop.tpl new file mode 100644 index 00000000..3d5d5275 --- /dev/null +++ b/charts/haproxy/templates/bin/_pre_stop.tpl @@ -0,0 +1,21 @@ +#!/bin/sh +{{/* +Copyright 2018 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -x + +touch /tmp/stop +sleep {{ .Values.conf.anchor.period }} diff --git a/charts/haproxy/templates/configmap-bin.yaml b/charts/haproxy/templates/configmap-bin.yaml new file mode 100644 index 00000000..367f0f2f --- /dev/null +++ b/charts/haproxy/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2018 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: haproxy-bin +data: + anchor.sh: | +{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + pre_stop.sh: | +{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/haproxy/templates/configmap-etc.yaml b/charts/haproxy/templates/configmap-etc.yaml new file mode 100644 index 00000000..33877bc3 --- /dev/null +++ b/charts/haproxy/templates/configmap-etc.yaml @@ -0,0 +1,35 @@ +{{/* +Copyright 2018 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: haproxy-etc +data: + haproxy.cfg.header: | + global + {{- range .Values.conf.haproxy.conf_parts.global }} + {{ . }} + {{- end }} + + stats socket /tmp/haproxy.sock mode 700 level admin expose-fd listeners + + defaults + {{- range .Values.conf.haproxy.conf_parts.defaults }} + {{ . }} + {{- end }} + haproxy.yaml: | +{{ tuple "etc/_haproxy.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/haproxy/templates/daemonset.yaml b/charts/haproxy/templates/daemonset.yaml new file mode 100644 index 00000000..aac66955 --- /dev/null +++ b/charts/haproxy/templates/daemonset.yaml @@ -0,0 +1,81 @@ +{{/* +Copyright 2018 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $envAll := . }} +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: haproxy-anchor +spec: +{{ tuple $envAll "haproxy_anchor" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "kubernetes" "haproxy-anchor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.haproxy_anchor.timeout }} + serviceAccountName: haproxy-anchor + containers: + - name: anchor + image: {{ .Values.images.tags.anchor }} + imagePullPolicy: {{ .Values.images.pull_policy }} + env: + - name: HAPROXY_HEADER + value: /tmp/etc/haproxy.cfg.header + - name: HAPROXY_CONF + value: /host{{ .Values.conf.haproxy.host_config_dir }}/haproxy.cfg + - name: NEXT_HAPROXY_CONF + value: "$(HAPROXY_CONF)-next" + - name: HAPROXY_SERVER_OPTS + value: {{ .Values.conf.haproxy.server_opts | quote }} + - name: KUBE_CA + value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - name: KUBE_TOKEN + value: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: KUBE_URL + value: {{ .Values.conf.anchor.kubernetes_url | quote }} + command: + - /tmp/bin/anchor.sh + lifecycle: + preStop: + exec: + command: + - /tmp/bin/pre_stop.sh + volumeMounts: + - name: haproxy-bin + mountPath: /tmp/bin + - name: haproxy-etc + mountPath: /tmp/etc + - name: host + mountPath: /host + volumes: + - name: haproxy-bin + configMap: + name: haproxy-bin + defaultMode: 0555 + - name: haproxy-etc + configMap: + name: haproxy-etc + defaultMode: 0444 + - name: host + hostPath: + path: / diff --git a/charts/haproxy/templates/etc/_haproxy.yaml.tpl b/charts/haproxy/templates/etc/_haproxy.yaml.tpl new file mode 100644 index 00000000..6a6230b5 --- /dev/null +++ b/charts/haproxy/templates/etc/_haproxy.yaml.tpl @@ -0,0 +1,88 @@ +{{/* +Copyright 2018 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +apiVersion: v1 +kind: Pod +metadata: + name: haproxy + namespace: {{ .Release.Namespace }} +spec: + hostNetwork: true + containers: + - name: haproxy + image: {{ .Values.images.tags.haproxy }} + imagePullPolicy: {{ .Values.images.pull_policy }} + hostNetwork: true + env: + - name: HAPROXY_CONF + value: {{ .Values.conf.haproxy.container_config_dir }}/haproxy.cfg + - name: LIVE_HAPROXY_CONF + value: /tmp/live_haproxy.cfg + command: + - /bin/sh + - -c + - | + set -eux + + while [ ! -s "$HAPROXY_CONF" ]; do + echo Waiting for "HAPROXY_CONF" + sleep 1 + done + echo vvv Starting with initial config vvv + cat "$HAPROXY_CONF" + echo + cp "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF" + + # NOTE(mark-burnett): sleep for clearer log output + sleep 1 + + haproxy -D -f "$LIVE_HAPROXY_CONF" -p /tmp/haproxy.pid + + echo HAProxy started, monitoring for config changes.. + + set +x + while true; do + if ! cmp -s "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"; then + echo vvv Replacing old config vvv + cat "$LIVE_HAPROXY_CONF" + echo + + echo vvv With new config vvv + cat "$HAPROXY_CONF" + echo + + cat "$HAPROXY_CONF" > "$LIVE_HAPROXY_CONF" + + # NOTE(mark-burnett): sleep for clearer log output + sleep 1 + + set -x + haproxy -D -f "$LIVE_HAPROXY_CONF" -p /tmp/haproxy.pid \ + -x /tmp/haproxy.sock \ + -sf $(cat /tmp/haproxy.pid) + set +x + fi + sleep {{ .Values.conf.haproxy.period }} + done + + volumeMounts: + - name: etc + mountPath: {{ .Values.conf.haproxy.container_config_dir }} + readOnly: True + volumes: + - name: etc + hostPath: + path: {{ .Values.conf.haproxy.host_config_dir }} diff --git a/charts/haproxy/templates/rbac.yaml b/charts/haproxy/templates/rbac.yaml new file mode 100644 index 00000000..e218a370 --- /dev/null +++ b/charts/haproxy/templates/rbac.yaml @@ -0,0 +1,52 @@ +{{/* +Copyright 2018 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: haproxy-anchor +{{- range $namespace, $services := $envAll.Values.conf.anchor.services }} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: haproxy-anchor + namespace: {{ $namespace }} +rules: + - apiGroups: [""] + resources: + - endpoints + verbs: + - get +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: haproxy-anchor + namespace: {{ $namespace }} +subjects: + - kind: ServiceAccount + name: haproxy-anchor + namespace: {{ $envAll.Release.Namespace }} + apiGroup: "" +roleRef: + kind: Role + name: haproxy-anchor + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/haproxy/values.yaml b/charts/haproxy/values.yaml new file mode 100644 index 00000000..3393eb60 --- /dev/null +++ b/charts/haproxy/values.yaml @@ -0,0 +1,88 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +conf: + anchor: + files_to_copy: + - source: /tmp/etc/haproxy.yaml + dest: /etc/kubernetes/manifests/haproxy.yaml + period: 30 + kubernetes_url: https://kubernetes.default:443 + services: + default: + kubernetes: + server_opts: "check" + conf_parts: + frontend: + - mode tcp + - option tcpka + - bind *:6553 + backend: + - mode tcp + - option tcpka + kube-system: + kubernetes-etcd: + server_opts: "check" + conf_parts: + frontend: + - mode tcp + - option tcpka + - bind *:2378 + backend: + - mode tcp + - option tcpka + + haproxy: + container_config_dir: /usr/local/etc/haproxy + host_config_dir: /etc/promenade/haproxy + period: 5 + conf_parts: + global: + - maxconn 10240 + defaults: + - timeout connect 5000ms + - timeout client 24h + - timeout server 24h + frontend: [] + backend: [] + +images: + tags: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6 + haproxy: haproxy:1.8.3 + pull_policy: "IfNotPresent" + +pod: + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + haproxy_anchor: + enabled: false + min_ready_seconds: 0 + max_unavailable: 1 + termination_grace_period: + haproxy_anchor: + timeout: 3600 + resources: + enabled: false + haproxy_anchor: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "2000m" + +release_group: null diff --git a/charts/promenade/templates/deployment-api.yaml b/charts/promenade/templates/deployment-api.yaml index a1b2e63d..df627a6a 100644 --- a/charts/promenade/templates/deployment-api.yaml +++ b/charts/promenade/templates/deployment-api.yaml @@ -46,11 +46,11 @@ spec: command: - /opt/promenade/entrypoint.sh - server - {{- if .Values.pod.env.promenade_api }} + {{- if $envAll.Values.pod.env.promenade_api }} env: - {{- range .Values.pod.env.promenade_api }} - - name: {{ .name }} - value: {{ .value }} + {{- range $envAll.Values.pod.env.promenade_api }} + - name: {{ .name | quote }} + value: {{ .value | quote }} {{- end }} {{- end }} ports: diff --git a/docs/source/configuration/host-system.rst b/docs/source/configuration/host-system.rst index 870bff26..b800be81 100644 --- a/docs/source/configuration/host-system.rst +++ b/docs/source/configuration/host-system.rst @@ -20,7 +20,7 @@ Sample Document tar_path: kubernetes/node/bin/kubelet mode: 0555 images: - coredns: coredns/coredns:011 + haproxy: haproxy:1.8.3 helm: helm: lachlanevenson/k8s-helm:v2.7.2 kubernetes: @@ -91,13 +91,15 @@ Core Images These images are used for essential functionality: -``coredns`` - coredns_ is configured and used for Kubernetes API discovery during +``haproxy`` + HAProxy_ is configured and used for Kubernetes API discovery during bootstrapping. ``kubectl`` Used for label application and validation tasks during bootstrapping. +.. _HAProxy: https://www.haproxy.org/ + Convenience Images ^^^^^^^^^^^^^^^^^^ @@ -105,9 +107,6 @@ Convenience Images The ``helm`` image is available for convenience. -.. _coredns: https://github.com/coredns/coredns - - Packages -------- diff --git a/docs/source/configuration/kubernetes-network.rst b/docs/source/configuration/kubernetes-network.rst index 7c442572..0f14d4c8 100644 --- a/docs/source/configuration/kubernetes-network.rst +++ b/docs/source/configuration/kubernetes-network.rst @@ -30,12 +30,15 @@ Sample Document - 8.8.4.4 kubernetes: + apiserver_port: 6443 + haproxy_port: 6553 pod_cidr: 10.97.0.0/16 service_cidr: 10.96.0.0/16 service_ip: 10.96.0.1 etcd: - service_ip: 10.96.0.2 + container_port: 2379 + haproxy_port: 2378 hosts_entries: - ip: 192.168.77.1 @@ -72,6 +75,13 @@ Kubernetes The ``kubernetes`` key contains: +``apiserver_port`` + The port that the Kubernetes API server process will listen on on hosts where it runs. + +``haproxy_port`` + The port that HAProxy will listen on on each host. This port will be used + by the ``kubelet`` and ``kube-proxy`` to find API servers in the cluster. + ``pod_cidr`` The CIDR from which the Kubernetes Controller Manager assigns pod IPs. diff --git a/entrypoint.sh b/entrypoint.sh index 9650099e..8a668575 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,14 +4,25 @@ set -ex PORT=${PORT:-9000} UWSGI_TIMEOUT=${UWSGI_TIMEOUT:-300} +PROMENADE_THREADS=${PROMENADE_THREADS:-1} +PROMENADE_WORKERS=${PROMENADE_WORKERS:-4} + if [ "$1" = 'server' ]; then exec uwsgi \ - --http :${PORT} \ - --http-timeout ${UWSGI_TIMEOUT} \ - -z ${UWSGI_TIMEOUT} \ + --http ":${PORT}" \ + --http-timeout "${UWSGI_TIMEOUT}" \ + --harakiri "${UWSGI_TIMEOUT}" \ + --socket-timeout "${UWSGI_TIMEOUT}" \ + --harakiri-verbose \ + --lazy-apps \ + --master \ + --thunder-lock \ + --die-on-term \ + -z "${UWSGI_TIMEOUT}" \ --paste config:/etc/promenade/api-paste.ini \ - --enable-threads -L \ - --workers 4 + --enable-threads \ + --threads "${PROMENADE_THREADS}" \ + --workers "${PROMENADE_WORKERS}" fi exec ${@} diff --git a/examples/basic/HostSystem.yaml b/examples/basic/HostSystem.yaml index b6c9bdc1..4f5ad665 100644 --- a/examples/basic/HostSystem.yaml +++ b/examples/basic/HostSystem.yaml @@ -30,7 +30,7 @@ data: rotate 1 } images: - coredns: coredns/coredns:0.9.9 + haproxy: haproxy:1.8.3 helm: helm: lachlanevenson/k8s-helm:v2.7.2 kubernetes: diff --git a/examples/basic/KubernetesNetwork.yaml b/examples/basic/KubernetesNetwork.yaml index 9c3d0373..1e35dbb8 100644 --- a/examples/basic/KubernetesNetwork.yaml +++ b/examples/basic/KubernetesNetwork.yaml @@ -21,12 +21,15 @@ data: - 8.8.4.4 kubernetes: + apiserver_port: 6443 + haproxy_port: 6553 pod_cidr: 10.97.0.0/16 service_cidr: 10.96.0.0/16 service_ip: 10.96.0.1 etcd: - service_ip: 10.96.0.2 + container_port: 2379 + haproxy_port: 2378 hosts_entries: - ip: 192.168.77.1 diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index f2c2e1fd..35cef7f6 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -69,6 +69,7 @@ metadata: data: description: Kubernetes components chart_group: + - haproxy - kubernetes-etcd - kubernetes-apiserver - kubernetes-controller-manager @@ -119,6 +120,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -160,7 +162,7 @@ data: tags: proxy: gcr.io/google_containers/hyperkube-amd64:v1.8.6 network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + kubernetes_netloc: 127.0.0.1:6553 source: type: local location: /etc/genesis/armada/assets/charts @@ -175,6 +177,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -419,6 +422,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -482,28 +486,6 @@ metadata: layeringDefinition: abstract: false layer: site - substitutions: - - - src: - schema: deckhand/CertificateAuthority/v1 - name: kubernetes - path: . - dest: - path: '.values.tls.ca' - - - src: - schema: deckhand/Certificate/v1 - name: coredns - path: . - dest: - path: '.values.tls.cert' - - - src: - schema: deckhand/CertificateKey/v1 - name: coredns - path: . - dest: - path: '.values.tls.key' data: chart_name: coredns release: coredns @@ -514,39 +496,9 @@ data: upgrade: no_hooks: true values: - coredns: - kubernetes_zones: - - cluster.local - - 10.96.0.0/16 - - 10.97.0.0/16 - upstream_nameservers: - - 8.8.8.8 - - 8.8.4.4 - - zones: - - name: promenade - services: - - bind_name: apiserver.kubernetes - service: - name: kubernetes-apiserver - namespace: kube-system - - bind_name: etcd.kubernetes - service: - name: kubernetes-etcd - namespace: kube-system - - bind_name: etcd.calico - service: - name: calico-etcd - namespace: kube-system images: - anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6 - coredns: coredns/coredns:0.9.9 - tls: - ca: placeholder - cert: placeholder - key: placeholder - network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + tags: + coredns: coredns/coredns:1.0.5 source: type: local location: /etc/genesis/armada/assets/charts @@ -555,6 +507,62 @@ data: - helm-toolkit --- schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: haproxy + layeringDefinition: + abstract: false + layer: site +data: + chart_name: haproxy + release: haproxy + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: + conf: + anchor: + kubernetes_url: https://kubernetes.default:443 + services: + default: + kubernetes: + server_opts: "check" + conf_parts: + frontend: + - mode tcp + - option tcpka + - bind *:6553 + backend: + - mode tcp + - option tcpka + kube-system: + kubernetes-etcd: + server_opts: "check" + conf_parts: + frontend: + - mode tcp + - option tcpka + - bind *:2378 + backend: + - mode tcp + - option tcpka + + images: + tags: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6 + haproxy: haproxy:1.8.3 + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: haproxy + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: kubernetes-apiserver @@ -626,7 +634,7 @@ data: values: apiserver: etcd: - endpoints: https://etcd.kubernetes.promenade:2379 + endpoints: https://127.0.0.1:2378 images: tags: anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6 @@ -716,7 +724,7 @@ data: cert: placeholder key: placeholder network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + kubernetes_netloc: 127.0.0.1:6553 pod_cidr: 10.97.0.0/16 service_cidr: 10.96.0.0/16 @@ -775,7 +783,7 @@ data: key: placeholder network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + kubernetes_netloc: 127.0.0.1:6553 images: tags: @@ -955,7 +963,7 @@ data: no_hooks: true values: anchor: - etcdctl_endpoint: 10.96.0.2 + etcdctl_endpoint: kubernetes-etcd.kube-system.svc.cluster.local labels: anchor: node_selector_key: kubernetes-etcd @@ -1012,7 +1020,6 @@ data: key: placeholder service: name: kubernetes-etcd - ip: 10.96.0.2 network: service_client: name: service_client @@ -1045,6 +1052,11 @@ data: wait: timeout: 600 values: + pod: + env: + promenade_api: + - name: PROMENADE_DEBUG + value: '1' conf: paste: app:promenade-api: diff --git a/examples/complete/HostSystem.yaml b/examples/complete/HostSystem.yaml index d3f7c2b0..921adc2e 100644 --- a/examples/complete/HostSystem.yaml +++ b/examples/complete/HostSystem.yaml @@ -31,7 +31,7 @@ data: rotate 1 } images: - coredns: coredns/coredns:0.9.9 + haproxy: haproxy:1.8.3 helm: helm: lachlanevenson/k8s-helm:v2.7.2 kubernetes: diff --git a/examples/complete/KubernetesNetwork.yaml b/examples/complete/KubernetesNetwork.yaml index b5755010..baa8d6f6 100644 --- a/examples/complete/KubernetesNetwork.yaml +++ b/examples/complete/KubernetesNetwork.yaml @@ -20,12 +20,15 @@ data: - 8.8.4.4 kubernetes: + apiserver_port: 6443 + haproxy_port: 6553 pod_cidr: 10.97.0.0/16 service_cidr: 10.96.0.0/16 service_ip: 10.96.0.1 etcd: - service_ip: 10.96.0.2 + container_port: 2379 + haproxy_port: 2378 hosts_entries: - ip: 192.168.77.1 diff --git a/examples/complete/armada-resources.yaml b/examples/complete/armada-resources.yaml index a9c166d8..bfe2dceb 100644 --- a/examples/complete/armada-resources.yaml +++ b/examples/complete/armada-resources.yaml @@ -86,6 +86,7 @@ metadata: data: description: Kubernetes components chart_group: + - haproxy - kubernetes-etcd - kubernetes-apiserver - kubernetes-controller-manager @@ -204,7 +205,7 @@ data: tags: proxy: gcr.io/google_containers/hyperkube-amd64:v1.8.6 network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + kubernetes_netloc: 127.0.0.1:6553 source: type: local location: /etc/genesis/armada/assets/charts @@ -528,29 +529,6 @@ metadata: layeringDefinition: abstract: false layer: site - storagePolicy: cleartext - substitutions: - - - src: - schema: deckhand/CertificateAuthority/v1 - name: kubernetes - path: . - dest: - path: '.values.tls.ca' - - - src: - schema: deckhand/Certificate/v1 - name: coredns - path: . - dest: - path: '.values.tls.cert' - - - src: - schema: deckhand/CertificateKey/v1 - name: coredns - path: . - dest: - path: '.values.tls.key' data: chart_name: coredns release: coredns @@ -561,39 +539,9 @@ data: upgrade: no_hooks: true values: - coredns: - kubernetes_zones: - - cluster.local - - 10.96.0.0/16 - - 10.97.0.0/16 - upstream_nameservers: - - 8.8.8.8 - - 8.8.4.4 - - zones: - - name: promenade - services: - - bind_name: apiserver.kubernetes - service: - name: kubernetes-apiserver - namespace: kube-system - - bind_name: etcd.kubernetes - service: - name: kubernetes-etcd - namespace: kube-system - - bind_name: etcd.calico - service: - name: calico-etcd - namespace: kube-system images: - anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6 - coredns: coredns/coredns:0.9.9 - tls: - ca: placeholder - cert: placeholder - key: placeholder - network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + tags: + coredns: coredns/coredns:1.0.5 source: type: local location: /etc/genesis/armada/assets/charts @@ -602,6 +550,62 @@ data: - helm-toolkit --- schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: haproxy + layeringDefinition: + abstract: false + layer: site +data: + chart_name: haproxy + release: haproxy + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: + conf: + anchor: + kubernetes_url: https://kubernetes.default:443 + services: + default: + kubernetes: + server_opts: "check" + conf_parts: + frontend: + - mode tcp + - option tcpka + - bind *:6553 + backend: + - mode tcp + - option tcpka + kube-system: + kubernetes-etcd: + server_opts: "check" + conf_parts: + frontend: + - mode tcp + - option tcpka + - bind *:2378 + backend: + - mode tcp + - option tcpka + + images: + tags: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6 + haproxy: haproxy:1.8.3 + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: haproxy + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: kubernetes-apiserver @@ -673,7 +677,7 @@ data: values: apiserver: etcd: - endpoints: https://etcd.kubernetes.promenade:2379 + endpoints: https://127.0.0.1:2378 images: tags: anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6 @@ -763,7 +767,7 @@ data: cert: placeholder key: placeholder network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + kubernetes_netloc: 127.0.0.1:6553 pod_cidr: 10.97.0.0/16 service_cidr: 10.96.0.0/16 @@ -822,7 +826,7 @@ data: key: placeholder network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + kubernetes_netloc: 127.0.0.1:6553 images: tags: @@ -1002,7 +1006,7 @@ data: no_hooks: true values: anchor: - etcdctl_endpoint: 10.96.0.2 + etcdctl_endpoint: kubernetes-etcd.kube-system.svc.cluster.local labels: anchor: node_selector_key: kubernetes-etcd @@ -1059,7 +1063,6 @@ data: key: placeholder service: name: kubernetes-etcd - ip: 10.96.0.2 network: service_client: name: service_client @@ -1756,6 +1759,11 @@ data: wait: timeout: 600 values: + pod: + env: + promenade_api: + - name: PROMENADE_DEBUG + value: '1' images: tags: dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 diff --git a/promenade/generator.py b/promenade/generator.py index 1e187be8..e00c4d1f 100644 --- a/promenade/generator.py +++ b/promenade/generator.py @@ -33,7 +33,7 @@ class Generator: ca='kubernetes', cn='apiserver', hosts=self._service_dns('kubernetes', 'default') + - ['localhost', '127.0.0.1', 'apiserver.kubernetes.promenade'] + + ['localhost', '127.0.0.1'] + [self.config['KubernetesNetwork:kubernetes.service_ip']]) self.gen( 'certificate', @@ -75,25 +75,19 @@ class Generator: cn='armada', groups=['system:masters']) - # Certificates for coredns - self.gen('certificate', 'coredns', ca='kubernetes', cn='coredns') - # Certificates for Kubernetes's etcd servers self.gen_etcd_certificates( ca='kubernetes-etcd', genesis=True, service_name='kubernetes-etcd', - service_namespace='kube-system', - service_ip=self.config['KubernetesNetwork:etcd.service_ip'], - additional_hosts=['etcd.kubernetes.promenade']) + service_namespace='kube-system') # Certificates for Calico's etcd servers self.gen_etcd_certificates( ca='calico-etcd', service_name='calico-etcd', service_namespace='kube-system', - service_ip=self.calico_etcd_service_ip, - additional_hosts=['etcd.calico.promenade']) + service_ip=self.calico_etcd_service_ip) # Certificates for Calico node self.gen( diff --git a/promenade/schemas/HostSystem.yaml b/promenade/schemas/HostSystem.yaml index 56629b6f..3bf35e3f 100644 --- a/promenade/schemas/HostSystem.yaml +++ b/promenade/schemas/HostSystem.yaml @@ -71,8 +71,11 @@ data: images: type: object properties: + # NOTE(mark-burnett): No longer used. coredns: $ref: '#/definitions/image' + haproxy: + $ref: '#/definitions/image' helm: type: object properties: @@ -90,7 +93,7 @@ data: - kubectl additionalProperties: false required: - - coredns + - haproxy - helm - kubernetes additionalProperties: false diff --git a/promenade/schemas/KubernetesNetwork.yaml b/promenade/schemas/KubernetesNetwork.yaml index d66fbc9b..7d53016a 100644 --- a/promenade/schemas/KubernetesNetwork.yaml +++ b/promenade/schemas/KubernetesNetwork.yaml @@ -52,10 +52,16 @@ data: etcd: type: object properties: + container_port: + type: integer + haproxy_port: + type: integer + # NOTE(mark-burnett): No longer used. service_ip: $ref: '#/definitions/ip_address' required: - - service_ip + - container_port + - haproxy_port additionalProperties: false kubernetes: @@ -67,10 +73,16 @@ data: $ref: '#/definitions/ip_address' service_cidr: $ref: '#/definitions/cidr' + apiserver_port: + type: integer + haproxy_port: + type: integer required: - pod_cidr - service_cidr - service_ip + - apiserver_port + - haproxy_port additionalProperties: false hosts_entries: type: array diff --git a/promenade/templates/roles/common/etc/coredns/Corefile b/promenade/templates/roles/common/etc/coredns/Corefile deleted file mode 100644 index cf4fdbbc..00000000 --- a/promenade/templates/roles/common/etc/coredns/Corefile +++ /dev/null @@ -1,13 +0,0 @@ -promenade { - file /etc/coredns/zones/promenade - errors stdout - log stdout -} - -. { - {%- if config['KubernetesNetwork:dns.upstream_servers'] is defined %} - proxy . {%- for server in config['KubernetesNetwork:dns.upstream_servers'] %} {{ server -}}{%- endfor %} - {%- endif %} - errors stdout - log stdout -} diff --git a/promenade/templates/roles/common/etc/coredns/zones/promenade b/promenade/templates/roles/common/etc/coredns/zones/promenade deleted file mode 100644 index 4b32f39a..00000000 --- a/promenade/templates/roles/common/etc/coredns/zones/promenade +++ /dev/null @@ -1,11 +0,0 @@ -$ORIGIN promenade. - -promenade. IN SOA @ root {{ now }} 3h 15m 1w 1d - -{%- if config['KubernetesNode:join_ip'] is defined %} -apiserver.kubernetes IN A {{ config['KubernetesNode:join_ip'] }} -{%- else %} -apiserver.kubernetes IN A 127.0.0.1 -{%- endif %} - -etcd.kubernetes IN A 127.0.0.1 diff --git a/promenade/templates/roles/common/etc/kubernetes/admin/kubeconfig.yaml b/promenade/templates/roles/common/etc/kubernetes/admin/kubeconfig.yaml index 87e9e634..e042b18c 100644 --- a/promenade/templates/roles/common/etc/kubernetes/admin/kubeconfig.yaml +++ b/promenade/templates/roles/common/etc/kubernetes/admin/kubeconfig.yaml @@ -2,7 +2,7 @@ apiVersion: v1 clusters: - cluster: - server: https://apiserver.kubernetes.promenade:6443 + server: https://127.0.0.1:6553 certificate-authority: pki/cluster-ca.pem name: kubernetes contexts: diff --git a/promenade/templates/roles/common/etc/kubernetes/kubeconfig b/promenade/templates/roles/common/etc/kubernetes/kubeconfig index 0e3dfaa8..85a93672 100644 --- a/promenade/templates/roles/common/etc/kubernetes/kubeconfig +++ b/promenade/templates/roles/common/etc/kubernetes/kubeconfig @@ -2,7 +2,7 @@ apiVersion: v1 clusters: - cluster: - server: https://apiserver.kubernetes.promenade:6443 + server: https://127.0.0.1:6553 certificate-authority: pki/cluster-ca.pem name: kubernetes contexts: diff --git a/promenade/templates/roles/common/etc/kubernetes/manifests/coredns.yaml b/promenade/templates/roles/common/etc/kubernetes/manifests/coredns.yaml deleted file mode 100644 index 3cd0a22c..00000000 --- a/promenade/templates/roles/common/etc/kubernetes/manifests/coredns.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: coredns - namespace: kube-system - labels: - # NOTE(mark-burnett): This is a host initialization Pod and should not be - # included in the Kubernetes DNS service, so ensure it does not get - # selected. - coredns-service: disabled - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' -spec: - hostNetwork: true - containers: - - name: coredns - image: {{ config['HostSystem:images.coredns'] }} - command: - - /coredns - - -conf - - /etc/coredns/Corefile - volumeMounts: - - name: config - mountPath: /etc/coredns - readOnly: true - volumes: - - name: config - hostPath: - path: /etc/coredns diff --git a/promenade/templates/roles/common/etc/kubernetes/manifests/haproxy.yaml b/promenade/templates/roles/common/etc/kubernetes/manifests/haproxy.yaml new file mode 100644 index 00000000..0ff79125 --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/manifests/haproxy.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: haproxy + namespace: kube-system + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' +spec: + hostNetwork: true + containers: + - name: haproxy + image: {{ config['HostSystem:images.haproxy'] }} + imagePullPolicy: IfNotPresent + hostNetwork: true + env: + - name: HAPROXY_CONF + value: /usr/local/etc/haproxy/haproxy.cfg + command: + - /bin/sh + - -c + - | + set -eux + + while [ ! -s "$HAPROXY_CONF" ]; do + echo Waiting for "HAPROXY_CONF" + sleep 1 + done + + haproxy -f "$HAPROXY_CONF" + + volumeMounts: + - name: etc + mountPath: /usr/local/etc/haproxy + readOnly: true + volumes: + - name: etc + hostPath: + path: /etc/promenade/haproxy diff --git a/promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg b/promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg new file mode 100644 index 00000000..bf304dae --- /dev/null +++ b/promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg @@ -0,0 +1,43 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +global + maxconn 10240 + +defaults + mode tcp + option tcpka + timeout connect 5000ms + timeout client 24h + timeout server 24h + +frontend default-kubernetes-fe + bind *:{{ config['KubernetesNetwork:kubernetes.haproxy_port'] }} + default_backend default-kubernetes-be + +{% set ip = config.get_first('KubernetesNode:join_ip', 'Genesis:ip') -%} + +backend default-kubernetes-be + option tcp-check + {%- set port = config['KubernetesNetwork:kubernetes.apiserver_port'] %} + server s{{ ip }} {{ ip }}:{{ port }} check port {{ port }} + +frontend kube-system-kubernetes-etcd-fe + bind *:{{ config['KubernetesNetwork:etcd.haproxy_port'] }} + default_backend kube-system-kubernetes-etcd-be + +backend kube-system-kubernetes-etcd-be + option tcp-check + {%- set port = config['KubernetesNetwork:etcd.container_port'] %} + server s{{ ip }} {{ ip }}:{{ port }} check port {{ port }} diff --git a/promenade/templates/roles/common/etc/resolv.conf b/promenade/templates/roles/common/etc/resolv.conf index 0c1d2687..faf63a1d 100644 --- a/promenade/templates/roles/common/etc/resolv.conf +++ b/promenade/templates/roles/common/etc/resolv.conf @@ -1,6 +1,6 @@ options timeout:1 attempts:1 -nameserver 127.0.0.1 +nameserver 10.96.0.10 {% for server in config['KubernetesNetwork:dns.upstream_servers'] | default([]) %} nameserver {{ server }} {%- endfor %} diff --git a/promenade/templates/roles/genesis/etc/genesis/controller-manager/kubeconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/controller-manager/kubeconfig.yaml index a6653470..8d8314b7 100644 --- a/promenade/templates/roles/genesis/etc/genesis/controller-manager/kubeconfig.yaml +++ b/promenade/templates/roles/genesis/etc/genesis/controller-manager/kubeconfig.yaml @@ -2,7 +2,7 @@ apiVersion: v1 clusters: - cluster: - server: https://apiserver.kubernetes.promenade:6443 + server: https://127.0.0.1:6553 certificate-authority: pki/cluster-ca.pem name: kubernetes contexts: diff --git a/promenade/templates/roles/genesis/etc/genesis/scheduler/kubeconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/scheduler/kubeconfig.yaml index e37c6c6c..cd81e434 100644 --- a/promenade/templates/roles/genesis/etc/genesis/scheduler/kubeconfig.yaml +++ b/promenade/templates/roles/genesis/etc/genesis/scheduler/kubeconfig.yaml @@ -2,7 +2,7 @@ apiVersion: v1 clusters: - cluster: - server: https://apiserver.kubernetes.promenade:6443 + server: https://127.0.0.1:6553 certificate-authority: pki/cluster-ca.pem name: kubernetes contexts: diff --git a/tools/g2/lib/registry.sh b/tools/g2/lib/registry.sh index e41b9a79..78eb01db 100644 --- a/tools/g2/lib/registry.sh +++ b/tools/g2/lib/registry.sh @@ -7,7 +7,7 @@ registry_down() { } registry_list_images() { - FILES=($(echo "$(config_configuration)" | xargs -n 1 -I DIRNAME find DIRNAME -type f -name '*.yaml')) + FILES=($(config_configuration | xargs -n 1 -I DIRNAME find DIRNAME -type f -name '*.yaml')) HOSTNAME_REGEX='[a-zA-Z0-9][a-zA-Z0-9_-]{0,62}' DOMAIN_NAME_REGEX="${HOSTNAME_REGEX}(\.${HOSTNAME_REGEX})*" diff --git a/tools/g2/lib/validate.sh b/tools/g2/lib/validate.sh index ab98bce5..8991b0fe 100644 --- a/tools/g2/lib/validate.sh +++ b/tools/g2/lib/validate.sh @@ -12,6 +12,8 @@ validate_etcd_membership() { shift 2 EXPECTED_MEMBERS="${*}" + # NOTE(mark-burnett): Wait a moment for disks in test environment to settle. + sleep 10 log Validating "${CLUSTER}" etcd membership via "${VM}" FOUND_MEMBERS=$(etcdctl_member_list "${CLUSTER}" "${VM}" | tr '\n' ' ' | sed 's/ $//') diff --git a/tools/g2/lib/virsh.sh b/tools/g2/lib/virsh.sh index 76d3a5d4..df436e9b 100644 --- a/tools/g2/lib/virsh.sh +++ b/tools/g2/lib/virsh.sh @@ -129,7 +129,7 @@ vm_create() { --cpu host \ --graphics vnc,listen=0.0.0.0 \ --noautoconsole \ - --network network=promenade \ + --network "network=promenade,model=virtio" \ --vcpus "$(config_vm_vcpus)" \ --memory "$(config_vm_memory)" \ --import \ diff --git a/tools/gate/config-templates/bootstrap-armada-config.yaml b/tools/gate/config-templates/bootstrap-armada-config.yaml index 58039738..22b02c0f 100644 --- a/tools/gate/config-templates/bootstrap-armada-config.yaml +++ b/tools/gate/config-templates/bootstrap-armada-config.yaml @@ -82,6 +82,7 @@ metadata: data: description: Kubernetes components chart_group: + - haproxy - kubernetes-etcd - kubernetes-apiserver - kubernetes-controller-manager @@ -123,23 +124,23 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: '$.values.secrets.tls.ca' + path: '.values.secrets.tls.ca' - src: schema: deckhand/Certificate/v1 name: proxy - path: $ + path: . dest: - path: '$.values.secrets.tls.cert' + path: '.values.secrets.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: proxy - path: $ + path: . dest: - path: '$.values.secrets.tls.key' + path: '.values.secrets.tls.key' data: chart_name: proxy release: kubernetes-proxy @@ -157,7 +158,7 @@ data: tags: proxy: ${IMAGE_HYPERKUBE} network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + kubernetes_netloc: 127.0.0.1:6553 source: type: local location: /etc/genesis/armada/assets/charts @@ -178,147 +179,147 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: calico-etcd - path: $ + path: . dest: - path: '$.values.secrets.tls.client.ca' + path: '.values.secrets.tls.client.ca' - src: schema: deckhand/CertificateAuthority/v1 name: calico-etcd-peer - path: $ + path: . dest: - path: '$.values.secrets.tls.peer.ca' + path: '.values.secrets.tls.peer.ca' - src: schema: deckhand/Certificate/v1 name: calico-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.cert' + path: '.values.secrets.anchor.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.key' + path: '.values.secrets.anchor.tls.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-${GENESIS_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.cert' + path: '.values.nodes[0].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-${GENESIS_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.key' + path: '.values.nodes[0].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-${GENESIS_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.cert' + path: '.values.nodes[0].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-${GENESIS_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.key' + path: '.values.nodes[0].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-${MASTER1_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.cert' + path: '.values.nodes[1].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-${MASTER1_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.key' + path: '.values.nodes[1].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-${MASTER1_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.cert' + path: '.values.nodes[1].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-${MASTER1_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.key' + path: '.values.nodes[1].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-${MASTER2_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.cert' + path: '.values.nodes[2].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-${MASTER2_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.key' + path: '.values.nodes[2].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-${MASTER2_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.cert' + path: '.values.nodes[2].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-${MASTER2_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.key' + path: '.values.nodes[2].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-${WORKER_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[3].tls.client.cert' + path: '.values.nodes[3].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-${WORKER_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[3].tls.client.key' + path: '.values.nodes[3].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-${WORKER_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[3].tls.peer.cert' + path: '.values.nodes[3].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-${WORKER_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[3].tls.peer.key' + path: '.values.nodes[3].tls.peer.key' data: chart_name: etcd @@ -421,23 +422,23 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: calico-etcd - path: $ + path: . dest: - path: '$.values.etcd.tls.ca' + path: '.values.etcd.tls.ca' - src: schema: deckhand/Certificate/v1 name: calico-node - path: $ + path: . dest: - path: '$.values.etcd.tls.cert' + path: '.values.etcd.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-node - path: $ + path: . dest: - path: '$.values.etcd.tls.key' + path: '.values.etcd.tls.key' data: chart_name: calico release: calico @@ -479,28 +480,6 @@ metadata: abstract: false layer: site storagePolicy: cleartext - substitutions: - - - src: - schema: deckhand/CertificateAuthority/v1 - name: kubernetes - path: $ - dest: - path: '$.values.tls.ca' - - - src: - schema: deckhand/Certificate/v1 - name: coredns - path: $ - dest: - path: '$.values.tls.cert' - - - src: - schema: deckhand/CertificateKey/v1 - name: coredns - path: $ - dest: - path: '$.values.tls.key' data: chart_name: coredns release: coredns @@ -509,39 +488,9 @@ data: upgrade: no_hooks: true values: - coredns: - kubernetes_zones: - - cluster.local - - 10.96.0.0/16 - - 10.97.0.0/16 - upstream_nameservers: - - 8.8.8.8 - - 8.8.4.4 - - zones: - - name: promenade - services: - - bind_name: apiserver.kubernetes - service: - name: kubernetes-apiserver - namespace: kube-system - - bind_name: etcd.kubernetes - service: - name: kubernetes-etcd - namespace: kube-system - - bind_name: etcd.calico - service: - name: calico-etcd - namespace: kube-system images: - anchor: ${IMAGE_HYPERKUBE} - coredns: ${IMAGE_COREDNS} - tls: - ca: placeholder - cert: placeholder - key: placeholder - network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + tags: + coredns: ${IMAGE_COREDNS} source: type: local location: /etc/genesis/armada/assets/charts @@ -550,6 +499,62 @@ data: - helm-toolkit --- schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: haproxy + layeringDefinition: + abstract: false + layer: site +data: + chart_name: haproxy + release: haproxy + namespace: kube-system + timeout: 600 + wait: + timeout: 600 + upgrade: + no_hooks: true + values: + conf: + anchor: + kubernetes_url: https://kubernetes.default:443 + services: + default: + kubernetes: + server_opts: "check" + conf_parts: + frontend: + - mode tcp + - option tcpka + - bind *:6553 + backend: + - mode tcp + - option tcpka + kube-system: + kubernetes-etcd: + server_opts: "check" + conf_parts: + frontend: + - mode tcp + - option tcpka + - bind *:2378 + backend: + - mode tcp + - option tcpka + + images: + tags: + anchor: ${IMAGE_HYPERKUBE} + haproxy: ${IMAGE_HAPROXY} + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: haproxy + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: kubernetes-apiserver @@ -562,52 +567,52 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: $.values.secrets.tls.ca + path: .values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: apiserver - path: $ + path: . dest: - path: $.values.secrets.tls.cert + path: .values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: apiserver - path: $ + path: . dest: - path: $.values.secrets.tls.key + path: .values.secrets.tls.key - src: schema: deckhand/CertificateAuthority/v1 name: kubernetes-etcd - path: $ + path: . dest: - path: $.values.secrets.etcd.tls.ca + path: .values.secrets.etcd.tls.ca - src: schema: deckhand/Certificate/v1 name: apiserver-etcd - path: $ + path: . dest: - path: $.values.secrets.etcd.tls.cert + path: .values.secrets.etcd.tls.cert - src: schema: deckhand/CertificateKey/v1 name: apiserver-etcd - path: $ + path: . dest: - path: $.values.secrets.etcd.tls.key + path: .values.secrets.etcd.tls.key - src: schema: deckhand/PublicKey/v1 name: service-account - path: $ + path: . dest: - path: $.values.secrets.service_account.public_key + path: .values.secrets.service_account.public_key data: chart_name: apiserver @@ -619,7 +624,7 @@ data: values: apiserver: etcd: - endpoints: https://etcd.kubernetes.promenade:2379 + endpoints: https://127.0.0.1:2378 images: tags: anchor: ${IMAGE_HYPERKUBE} @@ -661,31 +666,31 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: $.values.secrets.tls.ca + path: .values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: controller-manager - path: $ + path: . dest: - path: $.values.secrets.tls.cert + path: .values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: controller-manager - path: $ + path: . dest: - path: $.values.secrets.tls.key + path: .values.secrets.tls.key - src: schema: deckhand/PrivateKey/v1 name: service-account - path: $ + path: . dest: - path: $.values.secrets.service_account.private_key + path: .values.secrets.service_account.private_key data: chart_name: controller_manager @@ -707,7 +712,7 @@ data: cert: placeholder key: placeholder network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + kubernetes_netloc: 127.0.0.1:6553 pod_cidr: 10.97.0.0/16 service_cidr: 10.96.0.0/16 @@ -731,23 +736,23 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: $.values.secrets.tls.ca + path: .values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: scheduler - path: $ + path: . dest: - path: $.values.secrets.tls.cert + path: .values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: scheduler - path: $ + path: . dest: - path: $.values.secrets.tls.key + path: .values.secrets.tls.key data: chart_name: scheduler @@ -764,7 +769,7 @@ data: key: placeholder network: - kubernetes_netloc: apiserver.kubernetes.promenade:6443 + kubernetes_netloc: 127.0.0.1:6553 images: tags: @@ -791,118 +796,118 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes-etcd - path: $ + path: . dest: - path: '$.values.secrets.tls.client.ca' + path: '.values.secrets.tls.client.ca' - src: schema: deckhand/CertificateAuthority/v1 name: kubernetes-etcd-peer - path: $ + path: . dest: - path: '$.values.secrets.tls.peer.ca' + path: '.values.secrets.tls.peer.ca' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.cert' + path: '.values.secrets.anchor.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.key' + path: '.values.secrets.anchor.tls.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-${GENESIS_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.cert' + path: '.values.nodes[0].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-${GENESIS_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.key' + path: '.values.nodes[0].tls.client.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-${GENESIS_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.cert' + path: '.values.nodes[0].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-${GENESIS_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.key' + path: '.values.nodes[0].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-${MASTER1_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.cert' + path: '.values.nodes[1].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-${MASTER1_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.key' + path: '.values.nodes[1].tls.client.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-${MASTER1_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.cert' + path: '.values.nodes[1].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-${MASTER1_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.key' + path: '.values.nodes[1].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-${MASTER2_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.cert' + path: '.values.nodes[2].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-${MASTER2_HOSTNAME} - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.key' + path: '.values.nodes[2].tls.client.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-${MASTER2_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.cert' + path: '.values.nodes[2].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-${MASTER2_HOSTNAME}-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.key' + path: '.values.nodes[2].tls.peer.key' data: chart_name: etcd @@ -913,7 +918,7 @@ data: no_hooks: true values: anchor: - etcdctl_endpoint: 10.96.0.2 + etcdctl_endpoint: kubernetes-etcd labels: anchor: node_selector_key: kubernetes-etcd @@ -962,7 +967,6 @@ data: key: placeholder service: name: kubernetes-etcd - ip: 10.96.0.2 network: service_client: name: service_client diff --git a/tools/gate/config-templates/site-config.yaml b/tools/gate/config-templates/site-config.yaml index 373f1489..d3428238 100644 --- a/tools/gate/config-templates/site-config.yaml +++ b/tools/gate/config-templates/site-config.yaml @@ -21,12 +21,15 @@ data: - 8.8.4.4 kubernetes: + apiserver_port: 6443 + haproxy_port: 6553 pod_cidr: 10.97.0.0/16 service_cidr: 10.96.0.0/16 service_ip: 10.96.0.1 etcd: - service_ip: 10.96.0.2 + container_port: 2379 + haproxy_port: 2378 --- schema: promenade/Docker/v1 metadata: @@ -60,7 +63,7 @@ data: tar_path: kubernetes/node/bin/kubelet mode: 0555 images: - coredns: ${IMAGE_COREDNS} + haproxy: ${IMAGE_HAPROXY} helm: helm: ${IMAGE_HELM} kubernetes: diff --git a/tools/gate/default-config-env b/tools/gate/default-config-env index 6ef364f3..92e7f54d 100644 --- a/tools/gate/default-config-env +++ b/tools/gate/default-config-env @@ -3,9 +3,10 @@ IMAGE_CALICO_CNI=quay.io/calico/cni:v1.11.2 IMAGE_CALICO_CTL=quay.io/calico/ctl:v1.6.3 IMAGE_CALICO_KUBE_CONTROLLERS=quay.io/calico/kube-controllers:v1.0.2 IMAGE_CALICO_NODE=quay.io/calico/node:v2.6.5 -IMAGE_COREDNS=coredns/coredns:0.9.9 +IMAGE_COREDNS=coredns/coredns:1.0.5 IMAGE_DEP_CHECK=quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 IMAGE_ETCD=quay.io/coreos/etcd:v3.2.14 +IMAGE_HAPROXY=haproxy:1.8.3 IMAGE_HELM=lachlanevenson/k8s-helm:v2.7.2 IMAGE_HYPERKUBE=gcr.io/google_containers/hyperkube-amd64:v1.8.6 IMAGE_TILLER=gcr.io/kubernetes-helm/tiller:v2.7.2