diff --git a/.dockerignore b/.dockerignore index bcc01698..45429bd8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,6 @@ -__pycache__ -Vagrantfile .vagrant -promenade.tar -promenade.egg-info +Vagrantfile +__pycache__ +docs example -setup.sh +promenade.egg-info diff --git a/.gitignore b/.gitignore index c2254a24..c5d44895 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ __pycache__ /promenade.egg-info /tmp .tox/ +/.eggs ChangeLog # Sphinx documentation diff --git a/Dockerfile b/Dockerfile index cdbb6576..5ae070c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,11 +14,6 @@ FROM python:3.6 -ENV CNI_VERSION=v0.5.2 \ - HELM_VERSION=v2.5.0 \ - KUBECTL_VERSION=v1.6.8 \ - KUBELET_VERSION=v1.6.8 - VOLUME /etc/promenade VOLUME /target @@ -26,17 +21,6 @@ RUN mkdir /promenade WORKDIR /promenade RUN set -ex \ - && export BIN_DIR=/assets/usr/local/bin \ - && mkdir -p $BIN_DIR \ - && curl -Lo $BIN_DIR/kubelet https://storage.googleapis.com/kubernetes-release/release/$KUBELET_VERSION/bin/linux/amd64/kubelet \ - && curl -Lo $BIN_DIR/kubectl https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl \ - && chmod 555 $BIN_DIR/kubelet \ - && chmod 555 $BIN_DIR/kubectl \ - && mkdir -p /assets/opt/cni/bin \ - && curl -L https://github.com/containernetworking/cni/releases/download/$CNI_VERSION/cni-amd64-$CNI_VERSION.tgz | tar -zxv -C /assets/opt/cni/bin/ \ - && curl -L https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv -C /tmp linux-amd64/helm \ - && mv /tmp/linux-amd64/helm $BIN_DIR/helm \ - && chmod 555 $BIN_DIR/helm \ && curl -Lo /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \ && chmod 555 /usr/local/bin/cfssl \ && apt-get update -q \ @@ -48,7 +32,5 @@ RUN set -ex \ COPY requirements-frozen.txt /promenade RUN pip install --no-cache-dir -r requirements-frozen.txt -COPY ./assets/ /assets/ - COPY . /promenade RUN pip install -e /promenade diff --git a/Vagrantfile b/Vagrantfile index 76ea1968..f391131f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,7 +5,12 @@ Vagrant.configure("2") do |config| config.vm.box = "promenade/ubuntu1604" config.vm.box_check_update = false - config.vm.provision :shell, privileged: true, inline:< $MANIFEST_PATH +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.service.name }} + namespace: {{ .Release.Namespace }} + labels: + {{ .Values.service.name }}-service: enabled +spec: + hostNetwork: true + containers: + - name: apiserver + image: {{ .Values.images.apiserver }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + command: + - {{ .Values.apiserver.command }} + - --authorization-mode=Node,RBAC + - --advertise-address=\$(POD_IP) + - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds + - --anonymous-auth=false + - --bind-address=0.0.0.0 + - --secure-port={{ .Values.apiserver.port }} + - --insecure-port=0 + - --apiserver-count={{ .Values.apiserver.replicas }} + + - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem + - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem + - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem + + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem + - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem + - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem + + - --etcd-servers={{ .Values.apiserver.etcd.endpoints }} + - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem + - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem + - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem + + - --allow-privileged=true + + - --service-cluster-ip-range={{ .Values.network.service_cidr }} + - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub + + - --v=5 + + ports: + - containerPort: 443 + volumeMounts: + - name: etc + mountPath: /etc/kubernetes/apiserver + volumes: + - name: etc + hostPath: + path: {{ .Values.apiserver.host_etc_path }} +EODOC +} + +cleanup() { + rm -f $MANIFEST_PATH + rm -rf $PKI_PATH +} + +while true; do + if [ -e /tmp/stop ]; then + echo Stopping + cleanup + break + fi + + if [ ! -e $MANIFEST_PATH ]; then + copy_certificates + create_manifest + fi + + sleep {{ .Values.anchor.period }} +done diff --git a/charts/apiserver/templates/bin/_pre_stop.tpl b/charts/apiserver/templates/bin/_pre_stop.tpl new file mode 100644 index 00000000..f47fb9b9 --- /dev/null +++ b/charts/apiserver/templates/bin/_pre_stop.tpl @@ -0,0 +1,4 @@ +set -x + +touch /tmp/stop +sleep 10000 diff --git a/charts/apiserver/templates/configmap-bin.yaml b/charts/apiserver/templates/configmap-bin.yaml new file mode 100644 index 00000000..e2740643 --- /dev/null +++ b/charts/apiserver/templates/configmap-bin.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-bin +data: + anchor: |+ +{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + pre_stop: |+ +{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/apiserver/templates/configmap-certs.yaml b/charts/apiserver/templates/configmap-certs.yaml new file mode 100644 index 00000000..64bd1570 --- /dev/null +++ b/charts/apiserver/templates/configmap-certs.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-certs +data: + cluster-ca.pem: {{ .Values.tls.ca | quote }} + apiserver.pem: {{ .Values.tls.cert | quote }} + + etcd-client-ca.pem: {{ .Values.apiserver.etcd.tls.ca | quote }} + etcd-client.pem: {{ .Values.apiserver.etcd.tls.cert | quote }} + + service-account.pub: {{ .Values.service_account.public_key | quote }} diff --git a/charts/apiserver/templates/daemonset.yaml b/charts/apiserver/templates/daemonset.yaml new file mode 100644 index 00000000..aec34d95 --- /dev/null +++ b/charts/apiserver/templates/daemonset.yaml @@ -0,0 +1,67 @@ +--- +apiVersion: "extensions/v1beta1" +kind: DaemonSet +metadata: + name: {{ .Values.service.name }}-anchor + labels: + application: kubernetes + component: kubernetes-apiserver-anchor +spec: + selector: + matchLabels: + {{ .Values.service.name | quote }}: anchor + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + labels: + {{ .Values.service.name | quote }}: anchor + spec: + hostNetwork: true + dnsPolicy: {{ .Values.anchor.dns_policy }} + nodeSelector: + {{ .Values.anchor.node_selector.key }}: {{ .Values.anchor.node_selector.value }} + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: anchor + image: {{ .Values.images.anchor }} + command: + - /tmp/bin/anchor + lifecycle: + preStop: + exec: + command: + - /tmp/bin/pre_stop + volumeMounts: + - name: certs + mountPath: /certs + - name: host + mountPath: /host + - name: keys + mountPath: /keys + - name: bin + mountPath: /tmp/bin + terminationGracePeriodSeconds: {{ .Values.anchor.termination_grace_period }} + volumes: + - name: bin + configMap: + name: {{ .Values.service.name }}-bin + defaultMode: 0555 + - name: certs + configMap: + name: {{ .Values.service.name }}-certs + defaultMode: 0444 + - name: host + hostPath: + path: / + - name: keys + secret: + secretName: {{ .Values.service.name }}-keys + defaultMode: 0444 diff --git a/charts/apiserver/templates/secret-apiserver.yaml b/charts/apiserver/templates/secret-apiserver.yaml new file mode 100644 index 00000000..e2fbf779 --- /dev/null +++ b/charts/apiserver/templates/secret-apiserver.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.service.name }}-keys +type: Opaque +data: + apiserver-key.pem: {{ .Values.tls.key | b64enc }} + etcd-client-key.pem: {{ .Values.apiserver.etcd.tls.key | b64enc }} diff --git a/charts/apiserver/templates/service.yaml b/charts/apiserver/templates/service.yaml new file mode 100644 index 00000000..a24a4b2b --- /dev/null +++ b/charts/apiserver/templates/service.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.service.name }} +spec: + selector: + {{ .Values.service.name }}-service: enabled + {{- if .Values.service.ip }} + clusterIP: {{ .Values.service.ip }} + {{- end }} + ports: + - name: https + port: {{ .Values.apiserver.port }} diff --git a/charts/apiserver/values.yaml b/charts/apiserver/values.yaml new file mode 100644 index 00000000..45028c75 --- /dev/null +++ b/charts/apiserver/values.yaml @@ -0,0 +1,45 @@ +anchor: + dns_policy: Default + kubelet: + manifest_path: /etc/kubernetes/manifests + node_selector: + key: kubernetes-apiserver + value: enabled + termination_grace_period: 3600 + period: 15 + +apiserver: + command: /apiserver + + etcd: + endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local + tls: + ca: placeholder + cert: placeholder + key: placeholder + + host_etc_path: /etc/kubernetes/apiserver + + port: 6443 + replicas: 3 + +images: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + apiserver: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + +network: + kubernetes_service_ip: 10.96.0.1 + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + +service: + name: kubernetes-apiserver + ip: null + +service_account: + public_key: placeholder + +tls: + ca: placeholder + cert: placeholder + key: placeholder diff --git a/assets/etc/kubernetes/armada-loader/assets/charts/calico/.helmignore b/charts/calico/.helmignore similarity index 100% rename from assets/etc/kubernetes/armada-loader/assets/charts/calico/.helmignore rename to charts/calico/.helmignore diff --git a/assets/etc/kubernetes/armada-loader/assets/charts/calico/Chart.yaml b/charts/calico/Chart.yaml similarity index 100% rename from assets/etc/kubernetes/armada-loader/assets/charts/calico/Chart.yaml rename to charts/calico/Chart.yaml diff --git a/assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/configmap.yaml b/charts/calico/templates/configmap.yaml similarity index 100% rename from assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/configmap.yaml rename to charts/calico/templates/configmap.yaml diff --git a/assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/daemonset-node.yaml b/charts/calico/templates/daemonset-node.yaml similarity index 78% rename from assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/daemonset-node.yaml rename to charts/calico/templates/daemonset-node.yaml index debd228a..1bd00749 100644 --- a/assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/daemonset-node.yaml +++ b/charts/calico/templates/daemonset-node.yaml @@ -57,6 +57,9 @@ spec: configMapKeyRef: name: calico-config key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" @@ -95,9 +98,9 @@ spec: # Auto-detect the BGP IP address. - name: IP value: "" - {{- if .Values.calico.interface }} + {{- if .Values.calico.ip_autodetection_method }} - name: IP_AUTODETECTION_METHOD - value: interface={{ .Values.calico.interface }} + value: {{ .Values.calico.ip_autodetection_method }} {{- end }} securityContext: privileged: true @@ -144,6 +147,41 @@ spec: name: cni-net-dir - mountPath: /calico-secrets name: etcd-certs + {{- if .Values.calico.ctl.install_on_host }} + # This container installs calicoctl on each node. + - name: install-calicoctl + image: {{ .Values.images.cni }} + command: + - /bin/sh + - -c + - |- + set -ex + + cat <<'SCRIPT' > /target/calicoctl + #!/usr/bin/env bash + set -e + + exec docker run --rm -it \ + --net host \ + -e ETCD_CA_CERT_FILE=/etc/cni/net.d/calico-tls/etcd-ca \ + -e ETCD_CERT_FILE=/etc/cni/net.d/calico-tls/etcd-cert \ + -e ETCD_KEY_FILE=/etc/cni/net.d/calico-tls/etcd-key \ + -e ETCD_ENDPOINTS=https://{{ .Values.etcd.service.ip }}:{{ .Values.etcd.service.port }},https://127.0.0.1:{{ .Values.etcd.service.port }} \ + -v /etc/cni/net.d/calico-tls:/etc/cni/net.d/calico-tls \ + {{ .Values.images.ctl }} \ + $* + SCRIPT + + chmod 755 /target/calicoctl + + while true ; do + sleep 10000 + done + + volumeMounts: + - name: host-bin + mountPath: /target + {{- end }} volumes: # Used by calico/node. - name: lib-modules @@ -163,3 +201,8 @@ spec: - name: etcd-certs secret: secretName: calico-etcd-secrets + {{- if .Values.calico.ctl.install_on_host }} + - name: host-bin + hostPath: + path: /usr/local/bin + {{- end }} diff --git a/assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/deployment-policy-controller.yaml b/charts/calico/templates/deployment-policy-controller.yaml similarity index 100% rename from assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/deployment-policy-controller.yaml rename to charts/calico/templates/deployment-policy-controller.yaml diff --git a/assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/rbac.yaml b/charts/calico/templates/rbac.yaml similarity index 100% rename from assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/rbac.yaml rename to charts/calico/templates/rbac.yaml diff --git a/assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/secret.yaml b/charts/calico/templates/secret.yaml similarity index 74% rename from assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/secret.yaml rename to charts/calico/templates/secret.yaml index afe5550e..84169b64 100644 --- a/assets/etc/kubernetes/armada-loader/assets/charts/calico/templates/secret.yaml +++ b/charts/calico/templates/secret.yaml @@ -12,6 +12,6 @@ data: # not using TLS for etcd. # This self-hosted install expects three files with the following names. The values # should be base64 encoded strings of the entire contents of each file. - etcd-key: {{ .Values.calico.etcd.credentials.key | b64enc }} - etcd-cert: {{ .Values.calico.etcd.credentials.cert | b64enc }} - etcd-ca: {{ .Values.calico.etcd.credentials.ca | b64enc }} + etcd-key: {{ .Values.etcd.tls.key | b64enc }} + etcd-cert: {{ .Values.etcd.tls.cert | b64enc }} + etcd-ca: {{ .Values.etcd.tls.ca | b64enc }} diff --git a/charts/calico/values.yaml b/charts/calico/values.yaml new file mode 100644 index 00000000..9197bc74 --- /dev/null +++ b/charts/calico/values.yaml @@ -0,0 +1,23 @@ +calico: + ip_autodetection_method: null + pod_ip_cidr: 10.97.0.0/16 + ctl: + install_on_host: false + +etcd: + service: + ip: 10.96.232.136 + port: 6666 + tls: + ca: |- + invalid ca + cert: |- + invalid cert + key: |- + invalid key + +images: + cni: quay.io/calico/cni:v1.10.0 + ctl: quay.io/calico/ctl:v1.5.0 + node: quay.io/calico/node:v2.5.1 + policy_controller: quay.io/calico/kube-policy-controller:v0.7.0 diff --git a/charts/controller_manager/Chart.yaml b/charts/controller_manager/Chart.yaml new file mode 100644 index 00000000..2c67a387 --- /dev/null +++ b/charts/controller_manager/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A chart for Kubernetes controller-manager +name: controller_manager +version: 0.1.0 diff --git a/charts/controller_manager/requirements.yaml b/charts/controller_manager/requirements.yaml new file mode 100644 index 00000000..4b156320 --- /dev/null +++ b/charts/controller_manager/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/charts/controller_manager/templates/bin/_anchor.tpl b/charts/controller_manager/templates/bin/_anchor.tpl new file mode 100644 index 00000000..fc4584cf --- /dev/null +++ b/charts/controller_manager/templates/bin/_anchor.tpl @@ -0,0 +1,79 @@ +#!/bin/sh + +set -x + +export MANIFEST_PATH=/host{{ .Values.anchor.kubelet.manifest_path }}/{{ .Values.service.name }}.yaml +export ETC_PATH=/host{{ .Values.controller_manager.host_etc_path }} + +copy_etc_files() { + mkdir -p $ETC_PATH + cp /configmap/* /secret/* $ETC_PATH +} + +create_manifest() { + mkdir -p $(dirname $MANIFEST_PATH) + cat < $MANIFEST_PATH +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.service.name }} + namespace: {{ .Release.Namespace }} + labels: + {{ .Values.service.name }}-service: enabled +spec: + hostNetwork: true + containers: + - name: controller-manager + image: {{ .Values.images.controller_manager }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + command: + - {{ .Values.controller_manager.command }} + - --allocate-node-cidrs=true + - --cluster-cidr={{ .Values.network.pod_cidr }} + - --configure-cloud-routes=false + - --leader-elect=true + - --node-monitor-period={{ .Values.controller_manager.node_monitor_period }} + - --node-monitor-grace-period={{ .Values.controller_manager.node_monitor_grace_period }} + - --pod-eviction-timeout={{ .Values.controller_manager.pod_eviction_timeout }} + - --kubeconfig=/etc/kubernetes/controller-manager/kubeconfig.yaml + - --root-ca-file=/etc/kubernetes/controller-manager/cluster-ca.pem + - --service-account-private-key-file=/etc/kubernetes/controller-manager/service-account.priv + - --service-cluster-ip-range={{ .Values.network.service_cidr }} + - --use-service-account-credentials=true + + - --v=5 + + volumeMounts: + - name: etc + mountPath: /etc/kubernetes/controller-manager + volumes: + - name: etc + hostPath: + path: {{ .Values.controller_manager.host_etc_path }} +EODOC +} + +cleanup() { + rm -f $MANIFEST_PATH + rm -rf $ETC_PATH +} + +while true; do + if [ -e /tmp/stop ]; then + echo Stopping + cleanup + break + fi + + if [ ! -e $MANIFEST_PATH ]; then + copy_etc_files + create_manifest + fi + + sleep {{ .Values.anchor.period }} +done diff --git a/charts/controller_manager/templates/bin/_pre_stop.tpl b/charts/controller_manager/templates/bin/_pre_stop.tpl new file mode 100644 index 00000000..f47fb9b9 --- /dev/null +++ b/charts/controller_manager/templates/bin/_pre_stop.tpl @@ -0,0 +1,4 @@ +set -x + +touch /tmp/stop +sleep 10000 diff --git a/charts/controller_manager/templates/configmap-bin.yaml b/charts/controller_manager/templates/configmap-bin.yaml new file mode 100644 index 00000000..e2740643 --- /dev/null +++ b/charts/controller_manager/templates/configmap-bin.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-bin +data: + anchor: |+ +{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + pre_stop: |+ +{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/controller_manager/templates/configmap-etc.yaml b/charts/controller_manager/templates/configmap-etc.yaml new file mode 100644 index 00000000..27758516 --- /dev/null +++ b/charts/controller_manager/templates/configmap-etc.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-etc +data: + cluster-ca.pem: {{ .Values.tls.ca | quote }} + controller-manager.pem: {{ .Values.tls.cert | quote }} + kubeconfig.yaml: |+ + --- + apiVersion: v1 + clusters: + - cluster: + server: https://{{ .Values.network.kubernetes_netloc }} + certificate-authority: cluster-ca.pem + name: kubernetes + contexts: + - context: + cluster: kubernetes + user: controller-manager + name: controller-manager@kubernetes + current-context: controller-manager@kubernetes + kind: Config + preferences: {} + users: + - name: controller-manager + user: + client-certificate: controller-manager.pem + client-key: controller-manager-key.pem diff --git a/charts/controller_manager/templates/daemonset.yaml b/charts/controller_manager/templates/daemonset.yaml new file mode 100644 index 00000000..071206a3 --- /dev/null +++ b/charts/controller_manager/templates/daemonset.yaml @@ -0,0 +1,69 @@ +--- +apiVersion: "extensions/v1beta1" +kind: DaemonSet +metadata: + name: {{ .Values.service.name }}-anchor + labels: + application: kubernetes + component: kubernetes-controller-manager-anchor +spec: + selector: + matchLabels: + {{ .Values.service.name | quote }}: anchor + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + labels: + {{ .Values.service.name | quote }}: anchor + spec: + hostNetwork: true + dnsPolicy: {{ .Values.anchor.dns_policy }} + nodeSelector: + {{ .Values.anchor.node_selector.key }}: {{ .Values.anchor.node_selector.value }} + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: anchor + image: {{ .Values.images.anchor }} + command: + - /tmp/bin/anchor + lifecycle: + preStop: + exec: + command: + - /tmp/bin/pre_stop + volumeMounts: + - name: bin + mountPath: /tmp/bin + - name: etc + mountPath: /configmap + - name: host + mountPath: /host + - name: secret + mountPath: /secret + terminationGracePeriodSeconds: {{ .Values.anchor.termination_grace_period }} + volumes: + - name: bin + configMap: + name: {{ .Values.service.name }}-bin + defaultMode: 0555 + - name: etc + configMap: + name: {{ .Values.service.name }}-etc + defaultMode: 0444 + - name: host + hostPath: + path: / + - name: secret + secret: + secretName: {{ .Values.service.name }} + defaultMode: 0444 diff --git a/charts/controller_manager/templates/secret.yaml b/charts/controller_manager/templates/secret.yaml new file mode 100644 index 00000000..9b6af0ae --- /dev/null +++ b/charts/controller_manager/templates/secret.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.service.name }} +type: Opaque +data: + controller-manager-key.pem: {{ .Values.tls.key | b64enc }} + service-account.priv: {{ .Values.service_account.private_key | b64enc }} diff --git a/charts/controller_manager/values.yaml b/charts/controller_manager/values.yaml new file mode 100644 index 00000000..0b1b2ddd --- /dev/null +++ b/charts/controller_manager/values.yaml @@ -0,0 +1,36 @@ +anchor: + dns_policy: Default + kubelet: + manifest_path: /etc/kubernetes/manifests + node_selector: + key: kubernetes-controller-manager + value: enabled + period: 15 + termination_grace_period: 3600 + +controller_manager: + command: /controller-manager + host_etc_path: /etc/kubernetes/controller-manager + node_monitor_period: 5s + node_monitor_grace_period: 20s + pod_eviction_timeout: 60s + +service_account: + private_key: placeholder + +tls: + ca: placeholder + cert: placeholder + key: placeholder + +images: + anchor: quay.io/attcomdev/kube-controller-manager:v1.8.0 + controller_manager: quay.io/attcomdev/kube-controller-manager:v1.8.0 + +network: + kubernetes_netloc: 10.96.0.1 + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + +service: + name: kubernetes-controller-manager diff --git a/charts/coredns/Chart.yaml b/charts/coredns/Chart.yaml new file mode 100644 index 00000000..4aac9ced --- /dev/null +++ b/charts/coredns/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A chart for coredns +name: coredns +version: 0.1.0 diff --git a/charts/coredns/templates/bin/_anchor.tpl b/charts/coredns/templates/bin/_anchor.tpl new file mode 100644 index 00000000..83450669 --- /dev/null +++ b/charts/coredns/templates/bin/_anchor.tpl @@ -0,0 +1,129 @@ +#!/bin/sh + +{{- $envAll := . }} + +set -x + +export MANIFEST_PATH=/host{{ .Values.anchor.kubelet.manifest_path }}/{{ .Values.service.name }}.yaml +export ETC_PATH=/host{{ .Values.coredns.host_etc_path }} +TOKEN_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token +CA_CERT_PATH=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + +copy_etc_files() { + mkdir -p $ETC_PATH/zones + cp /configmap/* /secret/* $ETC_PATH + create_corefile +} + +create_corefile() { + cat < $ETC_PATH/Corefile +promenade { + file /etc/coredns/zones/promenade + loadbalance + errors stdout + log stdout +} + +{{ .Values.coredns.cluster_domain }} { + kubernetes { + endpoint https://{{ .Values.network.kubernetes_netloc }} + tls /etc/coredns/coredns.pem /etc/coredns/coredns-key.pem /etc/coredns/cluster-ca.pem + } + loadbalance + cache {{ .Values.coredns.cache.ttl }} + errors stdout + log stdout +} + +. { + {{- if .Values.coredns.upstream_nameservers }} + proxy . {{- range .Values.coredns.upstream_nameservers }} {{ . -}}{{- end }} + {{- end }} + errors stdout + log stdout +} +EOCOREFILE +} + +create_manifest() { + mkdir -p $(dirname $MANIFEST_PATH) +# XXX liveness/readiness probes + cat < $MANIFEST_PATH +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.service.name }} + namespace: {{ .Release.Namespace }} + labels: + {{ .Values.service.name }}-service: enabled + anchor-managed: enabled +spec: + hostNetwork: true + containers: + - name: coredns + image: {{ .Values.images.coredns }} + command: + - /coredns + - -conf + - /etc/coredns/Corefile + volumeMounts: + - name: etc + mountPath: /etc/coredns + volumes: + - name: etc + hostPath: + path: {{ .Values.coredns.host_etc_path }} +EODOC +} + +update_managed_zones() { +{{- range .Values.coredns.zones }} + +FILENAME="$ETC_PATH/zones/{{ .name }}" +NEXT_FILENAME="${FILENAME}-next" +SUCCESS=1 +NOW=$(date +%s) + +# Add Header +cat < $NEXT_FILENAME +\$ORIGIN {{ .name }}. +{{ .name }}. IN SOA @ root $NOW 3h 15m 1w 1d + +EOBIND +{{ range .services }} +# Don't accidentally log service account token +set +x +SERVICE_IPS=$(kubectl \ + --server https://{{ $envAll.Values.network.kubernetes_netloc }} \ + --certificate-authority $CA_CERT_PATH \ + --token $(cat $TOKEN_PATH) \ + -n {{ .service.namespace }} \ + get ep {{ .service.name }} \ + -o 'jsonpath={.subsets[*].addresses[*].ip}') +set -x +if [ "x$SERVICE_IPS" != "x" ]; then + for IP in $SERVICE_IPS; do + echo {{ .bind_name }} IN A $IP >> $NEXT_FILENAME + done +else + echo Failed to upate zone file for {{ .name }} + SUCCESS=0 +fi +{{- end }} + +if [ $SUCCESS = 1 ]; then + echo Replacing zone file $FILENAME + mv $NEXT_FILENAME $FILENAME +fi +{{- end }} +} + +copy_etc_files +create_manifest + +while true; do + update_managed_zones + + sleep {{ .Values.anchor.period }} +done diff --git a/charts/coredns/templates/configmap-bin.yaml b/charts/coredns/templates/configmap-bin.yaml new file mode 100644 index 00000000..3b76ec58 --- /dev/null +++ b/charts/coredns/templates/configmap-bin.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-bin +data: + anchor: |+ +{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/coredns/templates/configmap-etc.yaml b/charts/coredns/templates/configmap-etc.yaml new file mode 100644 index 00000000..54714212 --- /dev/null +++ b/charts/coredns/templates/configmap-etc.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-etc +data: + cluster-ca.pem: {{ .Values.tls.ca | quote }} + coredns.pem: {{ .Values.tls.cert | quote }} diff --git a/charts/coredns/templates/daemonset.yaml b/charts/coredns/templates/daemonset.yaml new file mode 100644 index 00000000..95825e84 --- /dev/null +++ b/charts/coredns/templates/daemonset.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: "extensions/v1beta1" +kind: DaemonSet +metadata: + name: {{ .Values.service.name }}-anchor + labels: + application: coredns + component: coredns-anchor +spec: + selector: + matchLabels: + {{ .Values.service.name | quote }}: anchor + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + {{ .Values.service.name | quote }}: anchor + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + hostNetwork: true + {{- if .Values.node_selector.key }} + nodeSelector: + {{ .Values.node_selector.key }}: {{ .Values.node_selector.value }} + {{- end }} + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + serviceAccountName: {{ .Values.service.name }} + containers: + - name: anchor + image: {{ .Values.images.anchor }} + command: + - /tmp/bin/anchor +# livenessProbe: +# httpGet: +# path: /healthz +# port: 10249 +# initialDelaySeconds: 15 +# periodSeconds: 15 +# failureThreshold: 3 +# readinessProbe: +# exec: +# command: +# - sh +# - -c +# - |- +# set -ex +# +# iptables-save | grep 'default/kubernetes:https' +# initialDelaySeconds: 15 +# periodSeconds: 15 + volumeMounts: + - name: bin + mountPath: /tmp/bin + - name: etc + mountPath: /configmap + - name: host + mountPath: /host + - name: secret + mountPath: /secret + volumes: + - name: bin + configMap: + name: {{ .Values.service.name }}-bin + defaultMode: 0555 + - name: etc + configMap: + name: {{ .Values.service.name }}-etc + defaultMode: 0444 + - name: host + hostPath: + path: / + - name: secret + secret: + secretName: {{ .Values.service.name }} + defaultMode: 0444 diff --git a/charts/coredns/templates/rbac.yaml b/charts/coredns/templates/rbac.yaml new file mode 100644 index 00000000..7bc65531 --- /dev/null +++ b/charts/coredns/templates/rbac.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: coredns + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" +subjects: + - kind: User + name: coredns + apiGroup: rbac.authorization.k8s.io + - kind: ServiceAccount + name: coredns + namespace: kube-system +roleRef: + kind: ClusterRole + name: system:coredns + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - get + - list + - watch diff --git a/charts/coredns/templates/secret.yaml b/charts/coredns/templates/secret.yaml new file mode 100644 index 00000000..1f6d585f --- /dev/null +++ b/charts/coredns/templates/secret.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.service.name }} +type: Opaque +data: + coredns-key.pem: {{ .Values.tls.key | b64enc }} diff --git a/charts/coredns/templates/service.yaml b/charts/coredns/templates/service.yaml new file mode 100644 index 00000000..621b8140 --- /dev/null +++ b/charts/coredns/templates/service.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.service.name }} +spec: + selector: + {{ .Values.service.name }}-service: enabled + clusterIP: {{ .Values.service.ip }} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/charts/coredns/values.yaml b/charts/coredns/values.yaml new file mode 100644 index 00000000..56406d00 --- /dev/null +++ b/charts/coredns/values.yaml @@ -0,0 +1,40 @@ +anchor: + kubelet: + manifest_path: /etc/kubernetes/manifests + node_selector: {} + period: 30 + +tls: + ca: placeholder + cert: placeholder + key: placeholder + +coredns: + cluster_domain: cluster.local + cache: + ttl: 60 + host_etc_path: /etc/coredns + upstream_nameservers: + - 8.8.8.8 + - 8.8.4.4 + + zones: + - name: promenade + services: + - bind_name: apiserver.kubernetes + service: + name: kubernetes + namespace: default + +images: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + coredns: coredns/coredns:011 + +network: + kubernetes_netloc: 10.96.0.1 + +node_selector: {} + +service: + name: coredns + ip: 10.96.0.10 diff --git a/charts/etcd/Chart.yaml b/charts/etcd/Chart.yaml new file mode 100644 index 00000000..78b042c7 --- /dev/null +++ b/charts/etcd/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A chart for a DaemonSet-based etcd deployment. +name: etcd +version: 0.1.0 diff --git a/charts/etcd/requirements.yaml b/charts/etcd/requirements.yaml new file mode 100644 index 00000000..4b156320 --- /dev/null +++ b/charts/etcd/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/charts/etcd/templates/bin/_etcdctl_anchor.tpl b/charts/etcd/templates/bin/_etcdctl_anchor.tpl new file mode 100644 index 00000000..9c7a696a --- /dev/null +++ b/charts/etcd/templates/bin/_etcdctl_anchor.tpl @@ -0,0 +1,198 @@ +#!/bin/sh + +set -x + +export CLIENT_ENDPOINT=https://$POD_IP:{{ .Values.service.client.target_port }} +export PEER_ENDPOINT=https://$POD_IP:{{ .Values.service.peer.target_port }} +export MANIFEST_PATH=/manifests/{{ .Values.service.name }}.yaml + +function copy_certificates { + ETCD_NAME=$1 + + set -e + + mkdir -p /etcd-etc/tls + # Copy CA Certificates in place + cp \ + /etc/etcd/tls/certs/client-ca.pem \ + /etc/etcd/tls/certs/peer-ca.pem \ + /etcd-etc/tls + + cp /etc/etcd/tls/certs/$ETCD_NAME-etcd-client.pem /etcd-etc/tls/etcd-client.pem + cp /etc/etcd/tls/certs/$ETCD_NAME-etcd-peer.pem /etcd-etc/tls/etcd-peer.pem + + cp /etc/etcd/tls/keys/$ETCD_NAME-etcd-client-key.pem /etcd-etc/tls/etcd-client-key.pem + cp /etc/etcd/tls/keys/$ETCD_NAME-etcd-peer-key.pem /etcd-etc/tls/etcd-peer-key.pem + + set +e +} + +function create_manifest { + ETCD_INITIAL_CLUSTER=$1 + ETCD_INITIAL_CLUSTER_STATE=$2 + cat < $MANIFEST_PATH +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.service.name }} + namespace: {{ .Release.Namespace }} + labels: + {{ .Values.service.name }}-service: enabled +spec: + hostNetwork: true + containers: + - name: etcd + image: {{ .Values.images.etcd }} + env: + - name: ETCD_NAME + value: $ETCD_NAME + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: ETCD_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_PEER_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_DATA_DIR + value: /var/lib/etcd + - name: ETCD_TRUSTED_CA_FILE + value: /etc/etcd/tls/client-ca.pem + - name: ETCD_CERT_FILE + value: /etc/etcd/tls/etcd-client.pem + - name: ETCD_STRICT_RECONFIG_CHECK + value: "true" + - name: ETCD_KEY_FILE + value: /etc/etcd/tls/etcd-client-key.pem + - name: ETCD_PEER_TRUSTED_CA_FILE + value: /etc/etcd/tls/peer-ca.pem + - name: ETCD_PEER_CERT_FILE + value: /etc/etcd/tls/etcd-peer.pem + - name: ETCD_PEER_KEY_FILE + value: /etc/etcd/tls/etcd-peer-key.pem + - name: ETCD_ADVERTISE_CLIENT_URLS + value: https://\$(POD_IP):{{ .Values.service.client.target_port }} + - name: ETCD_INITIAL_ADVERTISE_PEER_URLS + value: https://\$(POD_IP):{{ .Values.service.peer.target_port }} + - name: ETCD_INITIAL_CLUSTER_TOKEN + value: {{ .Values.service.name }}-init-token + - name: ETCD_LISTEN_CLIENT_URLS + value: https://0.0.0.0:{{ .Values.service.client.target_port }} + - name: ETCD_LISTEN_PEER_URLS + value: https://0.0.0.0:{{ .Values.service.peer.target_port }} + - name: ETCD_INITIAL_CLUSTER_STATE + value: $ETCD_INITIAL_CLUSTER_STATE + - name: ETCD_INITIAL_CLUSTER + value: $ETCD_INITIAL_CLUSTER + - name: ETCDCTL_API + value: '3' + - name: ETCDCTL_DIAL_TIMEOUT + value: 3s + - name: ETCDCTL_ENDPOINTS + value: https://127.0.0.1:{{ .Values.service.client.target_port }} + - name: ETCDCTL_CACERT + value: \$(ETCD_TRUSTED_CA_FILE) + - name: ETCDCTL_CERT + value: \$(ETCD_CERT_FILE) + - name: ETCDCTL_KEY + value: \$(ETCD_KEY_FILE) + volumeMounts: + - name: data + mountPath: /var/lib/etcd + - name: etc + mountPath: /etc/etcd + volumes: + - name: data + hostPath: + path: {{ .Values.etcd.host_data_path }} + - name: etc + hostPath: + path: {{ .Values.etcd.host_etc_path }} +... +EODOC +} + +while true; do + # TODO(mark-burnett) Need to monitor a file(s) when shutting down/starting + # up so I don't try to take two actions on the node at once. + {{- if .Values.bootstrapping.enabled }} + if [ -e /bootstrapping/{{ .Values.bootstrapping.filename }} ]; then + # If the first node is starting, wait for it to become healthy + end=$(($(date +%s) + {{ .Values.bootstrapping.timeout }})) + while etcdctl member list | grep $POD_IP; do + if ETCDCTL_ENDPOINTS=$CLIENT_ENDPOINT etcdctl endpoint health; then + echo Member appears healthy, removing bootstrap file. + rm /bootstrapping/{{ .Values.bootstrapping.filename }} + break + else + now=$(date +%s) + if [ $now -gt $end ]; then + echo Member did not start successfully before bootstrap timeout. Deleting and trying again. + rm -f $MANIFEST_PATH + sleep {{ .Values.anchor.period }} + break + fi + sleep {{ .Values.anchor.period }} + fi + done + fi + + if [ -e /bootstrapping/{{ .Values.bootstrapping.filename }} ]; then + # Bootstrap the first node + copy_certificates ${ETCD_NAME} + ETCD_INITIAL_CLUSTER=${ETCD_NAME}=$PEER_ENDPOINT + ETCD_INITIAL_CLUSTER_STATE=new + create_manifest $ETCD_INITIAL_CLUSTER $ETCD_INITIAL_CLUSTER_STATE + + continue + fi + {{- end }} + + sleep {{ .Values.anchor.period }} + + if [ -e /tmp/stopped ]; then + echo Stopping + break + fi + + if [ -e /tmp/stopping ]; then + echo Waiting to stop.. + continue + fi + + if [ ! -e $MANIFEST_PATH ]; then + if ! etcdctl member list > /tmp/members; then + echo Failed to locate existing cluster + continue + fi + + if ! grep $PEER_ENDPOINT /tmp/members; then + if grep -v '\bstarted\b' /tmp/members; then + echo Cluster does not appear fully online, waiting. + continue + fi + + # Add this member to the cluster + etcdctl member add $HOSTNAME --peer-urls $PEER_ENDPOINT + fi + + # If needed, drop the file in place + if [ ! -e FILE ]; then + # Refresh member list + etcdctl member list > /tmp/members + + if grep $PEER_ENDPOINT /tmp/members; then + copy_certificates ${ETCD_NAME} + + ETCD_INITIAL_CLUSTER=$(grep -v $PEER_ENDPOINT /tmp/members \ + | awk -F ', ' '{ print $3 "=" $4 }' \ + | tr '\n' ',' \ + | sed "s;\$;$ETCD_NAME=$PEER_ENDPOINT;") + ETCD_INITIAL_CLUSTER_STATE=existing + + create_manifest $ETCD_INITIAL_CLUSTER $ETCD_INITIAL_CLUSTER_STATE + fi + fi + fi +done diff --git a/charts/etcd/templates/bin/_pre_stop.tpl b/charts/etcd/templates/bin/_pre_stop.tpl new file mode 100644 index 00000000..d3d476bf --- /dev/null +++ b/charts/etcd/templates/bin/_pre_stop.tpl @@ -0,0 +1,35 @@ +#!/bin/sh + +set -x + +export PEER_ENDPOINT=https://$POD_IP:{{ .Values.service.peer.target_port }} +export MANIFEST_PATH=/manifests/{{ .Values.service.name }}.yaml + +function cleanup_host { + rm -f $MANIFEST_PATH + rm -rf /etcd-etc/tls/ + {{- if .Values.etcd.cleanup_data }} + rm -rf /etcd-data/* + {{- end }} +} + +# Let the anchor process know it should not try to start the server. +touch /tmp/stopping + +while true; do + if etcdctl member list > /tmp/stop_members; then + if grep $PEER_ENDPOINT /tmp/stop_members; then + # Find and remove the member from the cluster. + MEMBER_ID=$(grep $PEER_ENDPOINT /tmp/stop_members | awk -F ', ' '{ print $1 }') + etcdctl member remove $MEMBER_ID + else + cleanup_host + touch /tmp/stopped + exit 0 + fi + else + echo Failed to locate existing cluster + fi + + sleep {{ .Values.anchor.period }} +done diff --git a/charts/etcd/templates/bin/_readiness.tpl b/charts/etcd/templates/bin/_readiness.tpl new file mode 100644 index 00000000..028796b2 --- /dev/null +++ b/charts/etcd/templates/bin/_readiness.tpl @@ -0,0 +1,7 @@ +#!/bin/sh + +set -ex + +export ETCDCTL_ENDPOINTS=https://$POD_IP:{{ .Values.service.client.target_port }} + +etcdctl endpoint health diff --git a/charts/etcd/templates/configmap-bin.yaml b/charts/etcd/templates/configmap-bin.yaml new file mode 100644 index 00000000..fca0a0da --- /dev/null +++ b/charts/etcd/templates/configmap-bin.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-bin +data: + etcdctl_anchor: |+ +{{ tuple "bin/_etcdctl_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + pre_stop: |+ +{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + readiness: |+ +{{ tuple "bin/_readiness.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/etcd/templates/configmap-certs.yaml b/charts/etcd/templates/configmap-certs.yaml new file mode 100644 index 00000000..e3db28e8 --- /dev/null +++ b/charts/etcd/templates/configmap-certs.yaml @@ -0,0 +1,16 @@ +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $envAll.Values.service.name }}-certs +data: + anchor-etcd-client.pem: {{ $envAll.Values.anchor.tls.cert | quote }} + client-ca.pem: {{ $envAll.Values.tls.client.ca | quote }} + peer-ca.pem: {{ $envAll.Values.tls.peer.ca | quote }} +{{- range .Values.nodes }} + {{- $node := . }} + {{ $node.name }}-etcd-client.pem: {{ $node.tls.client.cert | quote }} + {{ $node.name }}-etcd-peer.pem: {{ $node.tls.peer.cert | quote }} +{{- end }} +... diff --git a/charts/etcd/templates/daemonset-anchor.yaml b/charts/etcd/templates/daemonset-anchor.yaml new file mode 100644 index 00000000..01b33d59 --- /dev/null +++ b/charts/etcd/templates/daemonset-anchor.yaml @@ -0,0 +1,111 @@ +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: {{ .Values.service.name }}-anchor +spec: + selector: + matchLabels: + {{ .Values.service.name | quote }}: anchor + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + labels: + {{ .Values.service.name | quote }}: anchor + spec: + hostNetwork: true + dnsPolicy: {{ .Values.anchor.dns_policy }} + nodeSelector: + {{ .Values.anchor.node_selector.key }}: {{ .Values.anchor.node_selector.value }} + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: etcdctl + image: {{ .Values.images.etcdctl }} + command: + - /tmp/etcdctl_anchor + env: + - name: ETCD_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: ETCDCTL_API + value: '3' + - name: ETCDCTL_DIAL_TIMEOUT + value: 3s + - name: ETCDCTL_ENDPOINTS + value: https://{{ .Values.anchor.etcdctl_endpoint }}:{{ .Values.service.client.port }} + - name: ETCDCTL_CACERT + value: /etc/etcd/tls/certs/client-ca.pem + - name: ETCDCTL_CERT + value: /etc/etcd/tls/certs/anchor-etcd-client.pem + - name: ETCDCTL_KEY + value: /etc/etcd/tls/keys/anchor-etcd-client-key.pem + readinessProbe: + exec: + command: + - /tmp/readiness + initialDelaySeconds: 15 + periodSeconds: 30 + lifecycle: + preStop: + exec: + command: + - /tmp/pre_stop + volumeMounts: + - name: bin + mountPath: /tmp + {{- if .Values.bootstrapping.enabled }} + - name: bootstrapping + mountPath: /bootstrapping + {{- end }} + - name: certs + mountPath: /etc/etcd/tls/certs + - name: etcd-etc + mountPath: /etcd-etc + {{- if .Values.etcd.cleanup_data }} + - name: etcd-data + mountPath: /etcd-data + {{- end }} + - name: keys + mountPath: /etc/etcd/tls/keys + - name: kubelet-manifests + mountPath: /manifests + terminationGracePeriodSeconds: {{ .Values.anchor.termination_grace_period }} + volumes: + - name: bin + configMap: + name: {{ .Values.service.name }}-bin + defaultMode: 0555 + {{- if .Values.bootstrapping.enabled }} + - name: bootstrapping + hostPath: + path: {{ .Values.bootstrapping.host_directory }} + {{- end }} + - name: certs + configMap: + name: {{ .Values.service.name }}-certs + defaultMode: 0444 + {{- if .Values.etcd.cleanup_data }} + - name: etcd-data + hostPath: + path: {{ .Values.etcd.host_data_path }} + {{- end }} + - name: keys + secret: + secretName: {{ .Values.service.name }}-keys + defaultMode: 0444 + - name: etcd-etc + hostPath: + path: {{ .Values.etcd.host_etc_path }} + - name: kubelet-manifests + hostPath: + path: {{ .Values.anchor.kubelet.manifest_path }} diff --git a/charts/etcd/templates/secret-keys.yaml b/charts/etcd/templates/secret-keys.yaml new file mode 100644 index 00000000..d06614a6 --- /dev/null +++ b/charts/etcd/templates/secret-keys.yaml @@ -0,0 +1,14 @@ +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $envAll.Values.service.name }}-keys +type: Opaque +data: + anchor-etcd-client-key.pem: {{ $envAll.Values.anchor.tls.key | b64enc }} +{{- range .Values.nodes }} + {{- $node := . }} + {{ $node.name }}-etcd-client-key.pem: {{ $node.tls.client.key | b64enc }} + {{ $node.name }}-etcd-peer-key.pem: {{ $node.tls.peer.key | b64enc }} +{{- end }} diff --git a/charts/etcd/templates/service.yaml b/charts/etcd/templates/service.yaml new file mode 100644 index 00000000..7c3a6213 --- /dev/null +++ b/charts/etcd/templates/service.yaml @@ -0,0 +1,16 @@ +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $envAll.Values.service.name }} +spec: + selector: + {{ $envAll.Values.service.name }}-service: enabled + {{- if $envAll.Values.service.ip }} + clusterIP: {{ $envAll.Values.service.ip }} + {{- end }} + ports: + - name: etcd + port: {{ $envAll.Values.service.client.port }} + targetPort: {{ $envAll.Values.service.client.target_port }} diff --git a/charts/etcd/values.yaml b/charts/etcd/values.yaml new file mode 100644 index 00000000..ab4e15cc --- /dev/null +++ b/charts/etcd/values.yaml @@ -0,0 +1,60 @@ +anchor: + dns_policy: Default + etcdctl_endpoint: example-etcd + host_data_path: /var/lib/etcd/example + + kubelet: + manifest_path: /etc/kubernetes/manifests + + node_selector: + key: etcd-example + value: enabled + + termination_grace_period: 3600 + period: 15 + + tls: + cert: placeholder + key: placeholder + +etcd: + host_etc_path: /etc/etcd-example + host_data_path: /var/lib/etcd/example + cleanup_data: true + +service: + name: example-etcd + ip: null + client: + port: 2379 + target_port: 2379 + peer: + port: 2380 + target_port: 2380 + +bootstrapping: + enabled: false + host_directory: /var/lib/anchor/etcd-example + filename: bootstrap + # XXX Can I just generalize to an anchor timeout? + timeout: 300 + +tls: + client: + ca: placeholder + peer: + ca: placeholder + +nodes: + - name: example-0 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + +images: + etcd: quay.io/coreos/etcd:v3.2.7 + etcdctl: quay.io/coreos/etcd:v3.2.7 diff --git a/charts/proxy/Chart.yaml b/charts/proxy/Chart.yaml new file mode 100644 index 00000000..a7933994 --- /dev/null +++ b/charts/proxy/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A chart for the Kubernetes proxy. +name: proxy +version: 0.1.0 diff --git a/charts/proxy/templates/configmap.yaml b/charts/proxy/templates/configmap.yaml new file mode 100644 index 00000000..d3af8910 --- /dev/null +++ b/charts/proxy/templates/configmap.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubernetes-proxy +data: + kubeconfig.yaml: |- + --- + apiVersion: v1 + clusters: + - cluster: + server: https://{{ .Values.network.kubernetes_netloc }} + certificate-authority: pki/cluster-ca.pem + name: kubernetes + contexts: + - context: + cluster: kubernetes + user: proxy + name: proxy@kubernetes + current-context: proxy@kubernetes + kind: Config + preferences: {} + users: + - name: proxy + user: + client-certificate: pki/proxy.pem + client-key: pki/proxy-key.pem + + cluster-ca.pem: {{ .Values.tls.ca | quote }} + proxy.pem: {{ .Values.tls.cert | quote }} diff --git a/charts/proxy/templates/daemonset.yaml b/charts/proxy/templates/daemonset.yaml new file mode 100644 index 00000000..9622099a --- /dev/null +++ b/charts/proxy/templates/daemonset.yaml @@ -0,0 +1,87 @@ +--- +apiVersion: "extensions/v1beta1" +kind: DaemonSet +metadata: + name: kubernetes-proxy + labels: + component: k8s-proxy +spec: + template: + metadata: + labels: + tier: node + component: k8s-proxy + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + hostNetwork: true + dnsPolicy: {{ .Values.dns_policy }} + {{- if .Values.node_selector.key }} + nodeSelector: + {{ .Values.node_selector.key }}: {{ .Values.node_selector.value }} + {{- end }} + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: proxy + image: {{ .Values.images.proxy }} + command: + - {{ .Values.proxy.command }} + - --cluster-cidr={{ .Values.network.pod_cidr }} + - --hostname-override=$(NODE_NAME) + - --kubeconfig=/etc/kubernetes/proxy/kubeconfig.yaml + - --proxy-mode=iptables + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: true +# livenessProbe: +# httpGet: +# path: /healthz +# port: 10249 +# initialDelaySeconds: 15 +# periodSeconds: 15 +# failureThreshold: 3 + readinessProbe: + exec: + command: + - sh + - -c + - |- + set -ex + + iptables-save | grep 'default/kubernetes:https' + initialDelaySeconds: 15 + periodSeconds: 15 + volumeMounts: + - name: proxy-cm + mountPath: /etc/kubernetes/proxy/kubeconfig.yaml + subPath: kubeconfig.yaml + - name: proxy-cm + mountPath: /etc/kubernetes/proxy/pki/proxy.pem + subPath: proxy.pem + - name: proxy-cm + mountPath: /etc/kubernetes/proxy/pki/cluster-ca.pem + subPath: cluster-ca.pem + - name: proxy-secret + mountPath: /etc/kubernetes/proxy/pki/proxy-key.pem + subPath: proxy-key.pem + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: proxy-cm + configMap: + name: kubernetes-proxy + - name: proxy-secret + secret: + secretName: kubernetes-proxy diff --git a/charts/proxy/templates/secret.yaml b/charts/proxy/templates/secret.yaml new file mode 100644 index 00000000..fe22824f --- /dev/null +++ b/charts/proxy/templates/secret.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: kubernetes-proxy +type: Opaque +data: + proxy-key.pem: {{ .Values.tls.key | b64enc }} diff --git a/charts/proxy/values.yaml b/charts/proxy/values.yaml new file mode 100644 index 00000000..e9b83ebf --- /dev/null +++ b/charts/proxy/values.yaml @@ -0,0 +1,19 @@ +dns_policy: Default +name: kubernetes-proxy + +tls: + ca: placeholder + cert: placeholder + key: placeholder + +proxy: + command: /proxy + +images: + proxy: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + +network: + kubernetes_netloc: 10.96.0.1 + pod_cidr: 10.97.0.0/16 + +node_selector: {} diff --git a/assets/etc/kubernetes/armada-loader/assets/charts/rbac/Chart.yaml b/charts/rbac/Chart.yaml similarity index 100% rename from assets/etc/kubernetes/armada-loader/assets/charts/rbac/Chart.yaml rename to charts/rbac/Chart.yaml diff --git a/assets/etc/kubernetes/armada-loader/assets/charts/rbac/templates/cluster-role-binding.yaml b/charts/rbac/templates/cluster-role-binding.yaml similarity index 86% rename from assets/etc/kubernetes/armada-loader/assets/charts/rbac/templates/cluster-role-binding.yaml rename to charts/rbac/templates/cluster-role-binding.yaml index c714f469..6325f5e3 100644 --- a/assets/etc/kubernetes/armada-loader/assets/charts/rbac/templates/cluster-role-binding.yaml +++ b/charts/rbac/templates/cluster-role-binding.yaml @@ -1,5 +1,5 @@ --- -apiVersion: rbac.authorization.k8s.io/v1alpha1 +apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: generous-permissions diff --git a/charts/rbac/values.yaml b/charts/rbac/values.yaml new file mode 100644 index 00000000..a3fc0897 --- /dev/null +++ b/charts/rbac/values.yaml @@ -0,0 +1 @@ +no: options diff --git a/charts/scheduler/Chart.yaml b/charts/scheduler/Chart.yaml new file mode 100644 index 00000000..f850d8a4 --- /dev/null +++ b/charts/scheduler/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A chart for Kubernetes scheduler. +name: scheduler +version: 0.1.0 diff --git a/charts/scheduler/templates/bin/_anchor.tpl b/charts/scheduler/templates/bin/_anchor.tpl new file mode 100644 index 00000000..1734fb2e --- /dev/null +++ b/charts/scheduler/templates/bin/_anchor.tpl @@ -0,0 +1,68 @@ +#!/bin/sh + +set -x + +export MANIFEST_PATH=/host{{ .Values.anchor.kubelet.manifest_path }}/{{ .Values.service.name }}.yaml +export ETC_PATH=/host{{ .Values.scheduler.host_etc_path }} + +copy_etc_files() { + mkdir -p $ETC_PATH + cp /configmap/* /secret/* $ETC_PATH +} + +create_manifest() { + mkdir -p $(dirname $MANIFEST_PATH) + cat < $MANIFEST_PATH +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Values.service.name }} + namespace: {{ .Release.Namespace }} + labels: + {{ .Values.service.name }}-service: enabled +spec: + hostNetwork: true + containers: + - name: scheduler + image: {{ .Values.images.scheduler }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + command: + - {{ .Values.scheduler.command }} + - --leader-elect=true + - --kubeconfig=/etc/kubernetes/scheduler/kubeconfig.yaml + - --v=5 + + volumeMounts: + - name: etc + mountPath: /etc/kubernetes/scheduler + volumes: + - name: etc + hostPath: + path: {{ .Values.scheduler.host_etc_path }} +EODOC +} + +cleanup() { + rm -f $MANIFEST_PATH + rm -rf $ETC_PATH +} + +while true; do + if [ -e /tmp/stop ]; then + echo Stopping + cleanup + break + fi + + if [ ! -e $MANIFEST_PATH ]; then + copy_etc_files + create_manifest + fi + + sleep {{ .Values.anchor.period }} +done diff --git a/charts/scheduler/templates/bin/_pre_stop.tpl b/charts/scheduler/templates/bin/_pre_stop.tpl new file mode 100644 index 00000000..f47fb9b9 --- /dev/null +++ b/charts/scheduler/templates/bin/_pre_stop.tpl @@ -0,0 +1,4 @@ +set -x + +touch /tmp/stop +sleep 10000 diff --git a/charts/scheduler/templates/configmap-bin.yaml b/charts/scheduler/templates/configmap-bin.yaml new file mode 100644 index 00000000..e2740643 --- /dev/null +++ b/charts/scheduler/templates/configmap-bin.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-bin +data: + anchor: |+ +{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + pre_stop: |+ +{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/scheduler/templates/configmap-etc.yaml b/charts/scheduler/templates/configmap-etc.yaml new file mode 100644 index 00000000..7a5ca093 --- /dev/null +++ b/charts/scheduler/templates/configmap-etc.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-etc +data: + kubeconfig.yaml: |- + --- + apiVersion: v1 + clusters: + - cluster: + server: https://{{ .Values.network.kubernetes_netloc }} + certificate-authority: cluster-ca.pem + name: kubernetes + contexts: + - context: + cluster: kubernetes + user: scheduler + name: scheduler@kubernetes + current-context: scheduler@kubernetes + kind: Config + preferences: {} + users: + - name: scheduler + user: + client-certificate: scheduler.pem + client-key: scheduler-key.pem + + cluster-ca.pem: {{ .Values.tls.ca | quote }} + scheduler.pem: {{ .Values.tls.cert | quote }} diff --git a/charts/scheduler/templates/daemonset.yaml b/charts/scheduler/templates/daemonset.yaml new file mode 100644 index 00000000..83048151 --- /dev/null +++ b/charts/scheduler/templates/daemonset.yaml @@ -0,0 +1,69 @@ +--- +apiVersion: "extensions/v1beta1" +kind: DaemonSet +metadata: + name: {{ .Values.service.name }}-anchor + labels: + application: kubernetes + component: kubernetes-scheduler-anchor +spec: + selector: + matchLabels: + {{ .Values.service.name | quote }}: anchor + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + labels: + {{ .Values.service.name | quote }}: anchor + spec: + hostNetwork: true + dnsPolicy: {{ .Values.anchor.dns_policy }} + nodeSelector: + {{ .Values.anchor.node_selector.key }}: {{ .Values.anchor.node_selector.value }} + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: anchor + image: {{ .Values.images.anchor }} + command: + - /tmp/bin/anchor + lifecycle: + preStop: + exec: + command: + - /tmp/bin/pre_stop + volumeMounts: + - name: bin + mountPath: /tmp/bin + - name: etc + mountPath: /configmap + - name: host + mountPath: /host + - name: secret + mountPath: /secret + terminationGracePeriodSeconds: {{ .Values.anchor.termination_grace_period }} + volumes: + - name: bin + configMap: + name: {{ .Values.service.name }}-bin + defaultMode: 0555 + - name: etc + configMap: + name: {{ .Values.service.name }}-etc + defaultMode: 0444 + - name: host + hostPath: + path: / + - name: secret + secret: + secretName: {{ .Values.service.name }} + defaultMode: 0444 diff --git a/charts/scheduler/templates/secret.yaml b/charts/scheduler/templates/secret.yaml new file mode 100644 index 00000000..43a9546f --- /dev/null +++ b/charts/scheduler/templates/secret.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.service.name }} +type: Opaque +data: + scheduler-key.pem: {{ .Values.tls.key | b64enc }} diff --git a/charts/scheduler/values.yaml b/charts/scheduler/values.yaml new file mode 100644 index 00000000..e3fdc90c --- /dev/null +++ b/charts/scheduler/values.yaml @@ -0,0 +1,28 @@ +anchor: + dns_policy: Default + kubelet: + manifest_path: /etc/kubernetes/manifests + node_selector: + key: kubernetes-scheduler + value: enabled + period: 15 + termination_grace_period: 3600 + +scheduler: + command: /scheduler + host_etc_path: /etc/kubernetes/scheduler + +service: + name: kubernetes-scheduler + +tls: + ca: placeholder + cert: placeholder + key: placeholder + +images: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + scheduler: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + +network: + kubernetes_netloc: 10.96.0.1 diff --git a/cleanup.sh b/cleanup.sh deleted file mode 100755 index c000e96c..00000000 --- a/cleanup.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -set -x - -export KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml -kubectl drain --force $(hostname) - -systemctl stop kubelet -docker rm -fv $(docker ps -aq) - -systemctl stop docker - -apt-get remove -qq -y dnsmasq ceph-common - -systemctl daemon-reload - -rm -rf \ - /etc/dnsmasq.d/kubernetes-masters \ - /etc/dnsmasq.d/upstream-dns \ - /etc/docker \ - /etc/kubernetes \ - /etc/systemd/system/docker.service.d \ - /etc/systemd/system/kubelet \ - /opt/cni \ - /usr/local/bin/bootstrap \ - /usr/local/bin/helm \ - /usr/local/bin/kubectl \ - /usr/local/bin/kubelet \ - /var/lib/auxiliary-etcd-0 \ - /var/lib/auxiliary-etcd-1 \ - /var/lib/kube-etcd \ - /var/lib/prom.done diff --git a/docs/source/conf.py b/docs/source/conf.py index 45d1849d..4d60f1b8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -99,7 +99,9 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# NOTE(mark-burnett): Currently, we don't have any static files and the +# non-existence of this directory causes a sphinx exception. +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst deleted file mode 100644 index 6b88a5e4..00000000 --- a/docs/source/configuration.rst +++ /dev/null @@ -1,165 +0,0 @@ -Promenade Configuration -======================= - -Promenade is configured using a set Kubernetes-like YAML documents. Many of -these documents can be automatically derived from a few core configuration -documents or generated automatically (e.g. certificates). All of these -documents can be specified in detail allowing for fine-grained control over -cluster deployment. - -Generally, these documents have the following form: - -.. code-block:: yaml - - --- - apiVersion: promenade/v1 - kind: Kind - metadata: - compliant: metadata - spec: - detailed: data - -``apiVersion`` identifies the document as Promenade configuration. Currently -only ``promenade/v1`` is supported. - -``kind`` describe the detailed type of document. Valid kinds are: - -- ``Certificate`` - An x509 certificate. -- ``CertificateAuthority`` - An x509 certificate authority certificate. -- ``CertificateAuthorityKey`` - The private key for a certificate authority. -- ``CertificateKey`` - The private key for a certificate. -- ``Cluster`` - Cluster configuration containing node host names, IPs & roles. -- ``Etcd`` - Specific configuration for an etcd cluster. -- ``Masters`` - Host names and IPs of master nodes. -- ``Network`` - Configuration details for Kubernetes networking components. -- ``Node`` - Specific configuration for a single host. -- ``PrivateKey`` - A private key, e.g. the ``controller-manager``'s token signing key. -- ``PublicKey`` - A public key, e.g. the key for verifying service account tokens. -- ``Versions`` - Specifies versions of packages and images to be deployed. - -``metadata`` are used to select specific documents of a given ``kind``. For -example, the various services must each select their specific ``Certificate``s. -``metadata`` are also used by Drydock to select the configuration files that are -needed for a particular node. - -``spec`` contains specific data for each kind of configuration document. - -Additionally, documents for [Armada](https://github.com/att-comdev/armada) are -allowed and will be applied after CNI and DNS are deployed. - -Generating Configuration from Minimal Input -------------------------------------------- - -To construct a complete set of cluster configuration, the minimal input are -``Cluster``, ``Network`` and ``Versions`` documents. To see complete examples of -these, please see the [example](example/vagrant-input-config.yaml). - -The ``Cluster`` configuration must contain an entry for each host for which -configuration should be generated. Each host must contain an ``ip``, and -optionally ``roles`` and ``additional_labels``. Valid ``roles`` are currently -``genesis`` and ``master``. ``additional_labels`` are Kubernetes labels which will -be added to the node. - -Here's an example ``Cluster`` document: - -.. code-block:: yaml - - apiVersion: promenade/v1 - kind: Cluster - metadata: - name: example - target: none - spec: - nodes: - n0: - ip: 192.168.77.10 - roles: - - master - - genesis - additional_labels: - - beta.kubernetes.io/arch=amd64 - -The ``Network`` document must contain: - -- ````cluster_domain```` - The domain for the cluster, e.g. ``cluster.local``. -- ````cluster_dns```` - The IP of the cluster DNS,e .g. ``10.96.0.10``. -- ``kube_service_ip`` - The IP of the ``kubernetes`` service, e.g. ``10.96.0.1``. -- ``pod_ip_cidr`` - The CIDR from which pod IPs will be assigned, e.g. ``10.97.0.0/16``. -- ``service_ip_cidr`` - The CIDR from which service IPs will be assigned, e.g. ``10.96.0.0/16``. -- ``etcd_service_ip`` - The IP address of the ``etcd`` service, e.g. ``10.96.232.136``. -- ``dns_servers`` - A list of upstream DNS server IPs. - -Optionally, proxy settings can be specified here as well. These should all -generally be set together: ``http_proxy``, ``https_proxy``, ``no_proxy``. ``no_proxy`` -must include all master IP addresses, and the ``kubernetes`` service name. - -Here's an example ``Network`` document: - -.. code-block:: yaml - - apiVersion: promenade/v1 - kind: Network - metadata: - cluster: example - name: example - target: all - spec: - cluster_domain: cluster.local - cluster_dns: 10.96.0.10 - kube_service_ip: 10.96.0.1 - pod_ip_cidr: 10.97.0.0/16 - service_ip_cidr: 10.96.0.0/16 - etcd_service_ip: 10.96.232.136 - dns_servers: - - 8.8.8.8 - - 8.8.4.4 - http_proxy: http://proxy.example.com:8080 - https_proxy: http://proxy.example.com:8080 - no_proxy: 192.168.77.10,192.168.77.11,192.168.77.12,127.0.0.1,kubernetes,kubernetes.default.svc.cluster.local - -The ``Versions`` document must define the Promenade image to be used and the -Docker package version. Currently, only the versions specified for these two -items are respected. - -Here's an example ``Versions`` document: - -.. code-block:: yaml - - apiVersion: promenade/v1 - kind: Versions - metadata: - cluster: example - name: example - target: all - spec: - images: - promenade: quay.io/attcomdev/promenade:latest - packages: - docker: docker.io=1.12.6-0ubuntu1~16.04.1 - -Given these documents (see the [example](example/vagrant-input-config.yaml)), -Promenade can derive the remaining configuration and generate certificates and -keys using the following command: - -.. code-block:: bash - - mkdir -p configs - docker run --rm -t \ - -v $(pwd):/target \ - quay.io/attcomdev/promenade:latest \ - promenade -v generate \ - -c /target/example/vagrant-input-config.yaml \ - -o /target/configs - -This will generate the following files in the ``configs`` directory: - -- ``up.sh`` - A script which will bring up a node to create or join a cluster. -- ``admin-bundle.yaml`` - A collection of generated certificates, private keys - and core configuration. -- ``complete-bundle.yaml`` - A set of generated documents suitable for upload - into Drydock for future delivery to nodes to be provisioned to join the - cluster. - -Additionally, a YAML file for each host described in the ``Cluster`` document -will be placed here. These files each contain every document needed for that -particular node to create or join the cluster. diff --git a/docs/source/configuration/docker.rst b/docs/source/configuration/docker.rst new file mode 100644 index 00000000..591a0e54 --- /dev/null +++ b/docs/source/configuration/docker.rst @@ -0,0 +1,29 @@ +Docker +====== + +Configuration for the docker daemon. This document contains a single `config` +key that directly translates into the contents of the `daemon.json` file +described in `Docker's configuration`_. + + +Sample Document +--------------- + +Here is a sample document: + +.. code-block:: yaml + + schema: promenade/Docker/v1 + metadata: + schema: metadata/Document/v1 + name: docker + layeringDefinition: + abstract: false + layer: site + data: + config: + live-restore: true + storage-driver: overlay2 + + +.. _Docker's configuration: https://docs.docker.com/engine/reference/commandline/dockerd/ diff --git a/docs/source/configuration/genesis.rst b/docs/source/configuration/genesis.rst new file mode 100644 index 00000000..50d9256d --- /dev/null +++ b/docs/source/configuration/genesis.rst @@ -0,0 +1,66 @@ +Genesis +======= + +Specific configuration for the genesis process. This document is a strict +superset of the combination of :doc:`kubernetes-node` and :doc:`host-system`, +so only differences are discussed here. + + +Sample Document +--------------- + +Here is a complete sample document: + +.. code-block:: yaml + + schema: promenade/Genesis/v1 + metadata: + schema: metadata/Document/v1 + name: genesis + layeringDefinition: + abstract: false + layer: site + data: + hostname: n0 + ip: 192.168.77.10 + labels: + static: + - calico-etcd=enabled + - node-role.kubernetes.io/master= + dynamic: + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - promenade-genesis=enabled + - ucp-control-plane=enabled + images: + armada: quay.io/attcomdev/armada:latest + helm: + tiller: gcr.io/kubernetes-helm/tiller:v2.5.1 + kubernetes: + apiserver: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + controller-manager: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + etcd: quay.io/coreos/etcd:v3.0.17 + scheduler: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + files: + - path: /var/lib/anchor/calico-etcd-bootstrap + content: "" + mode: 0644 + + +Bootstrapping Images +-------------------- + +Bootstrapping images are specified in the top level key ``images``: + +.. code-block:: yaml + + armada: + helm: + tiller: + kubernetes: + apiserver: + controller-manager: + etcd: + scheduler: diff --git a/docs/source/configuration/host-system.rst b/docs/source/configuration/host-system.rst new file mode 100644 index 00000000..22f5b2c3 --- /dev/null +++ b/docs/source/configuration/host-system.rst @@ -0,0 +1,133 @@ +HostSystem +========== + +Sample Document +--------------- + +.. code-block:: yaml + + schema: promenade/HostSystem/v1 + metadata: + schema: metadata/Document/v1 + name: host-system + layeringDefinition: + abstract: false + layer: site + data: + files: + - path: /opt/kubernetes/bin/kubelet + tar_url: https://dl.k8s.io/v1.8.0/kubernetes-node-linux-amd64.tar.gz + tar_path: kubernetes/node/bin/kubelet + mode: 0555 + images: + coredns: coredns/coredns:011 + helm: + helm: lachlanevenson/k8s-helm:v2.5.1 + kubernetes: + kubectl: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + packages: + repositories: + - deb http://apt.dockerproject.org/repo ubuntu-xenial main + keys: + - |- + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFWln24BEADrBl5p99uKh8+rpvqJ48u4eTtjeXAWbslJotmC/CakbNSqOb9o + ddfzRvGVeJVERt/Q/mlvEqgnyTQy+e6oEYN2Y2kqXceUhXagThnqCoxcEJ3+KM4R + mYdoe/BJ/J/6rHOjq7Omk24z2qB3RU1uAv57iY5VGw5p45uZB4C4pNNsBJXoCvPn + TGAs/7IrekFZDDgVraPx/hdiwopQ8NltSfZCyu/jPpWFK28TR8yfVlzYFwibj5WK + dHM7ZTqlA1tHIG+agyPf3Rae0jPMsHR6q+arXVwMccyOi+ULU0z8mHUJ3iEMIrpT + X+80KaN/ZjibfsBOCjcfiJSB/acn4nxQQgNZigna32velafhQivsNREFeJpzENiG + HOoyC6qVeOgKrRiKxzymj0FIMLru/iFF5pSWcBQB7PYlt8J0G80lAcPr6VCiN+4c + NKv03SdvA69dCOj79PuO9IIvQsJXsSq96HB+TeEmmL+xSdpGtGdCJHHM1fDeCqkZ + hT+RtBGQL2SEdWjxbF43oQopocT8cHvyX6Zaltn0svoGs+wX3Z/H6/8P5anog43U + 65c0A+64Jj00rNDr8j31izhtQMRo892kGeQAaaxg4Pz6HnS7hRC+cOMHUU4HA7iM + zHrouAdYeTZeZEQOA7SxtCME9ZnGwe2grxPXh/U/80WJGkzLFNcTKdv+rwARAQAB + tDdEb2NrZXIgUmVsZWFzZSBUb29sIChyZWxlYXNlZG9ja2VyKSA8ZG9ja2VyQGRv + Y2tlci5jb20+iQI4BBMBAgAiBQJVpZ9uAhsvBgsJCAcDAgYVCAIJCgsEFgIDAQIe + AQIXgAAKCRD3YiFXLFJgnbRfEAC9Uai7Rv20QIDlDogRzd+Vebg4ahyoUdj0CH+n + Ak40RIoq6G26u1e+sdgjpCa8jF6vrx+smpgd1HeJdmpahUX0XN3X9f9qU9oj9A4I + 1WDalRWJh+tP5WNv2ySy6AwcP9QnjuBMRTnTK27pk1sEMg9oJHK5p+ts8hlSC4Sl + uyMKH5NMVy9c+A9yqq9NF6M6d6/ehKfBFFLG9BX+XLBATvf1ZemGVHQusCQebTGv + 0C0V9yqtdPdRWVIEhHxyNHATaVYOafTj/EF0lDxLl6zDT6trRV5n9F1VCEh4Aal8 + L5MxVPcIZVO7NHT2EkQgn8CvWjV3oKl2GopZF8V4XdJRl90U/WDv/6cmfI08GkzD + YBHhS8ULWRFwGKobsSTyIvnbk4NtKdnTGyTJCQ8+6i52s+C54PiNgfj2ieNn6oOR + 7d+bNCcG1CdOYY+ZXVOcsjl73UYvtJrO0Rl/NpYERkZ5d/tzw4jZ6FCXgggA/Zxc + jk6Y1ZvIm8Mt8wLRFH9Nww+FVsCtaCXJLP8DlJLASMD9rl5QS9Ku3u7ZNrr5HWXP + HXITX660jglyshch6CWeiUATqjIAzkEQom/kEnOrvJAtkypRJ59vYQOedZ1sFVEL + MXg2UCkD/FwojfnVtjzYaTCeGwFQeqzHmM241iuOmBYPeyTY5veF49aBJA1gEJOQ + TvBR8Q== + =Fm3p + -----END PGP PUBLIC KEY BLOCK----- + additional: + - curl + - jq + required: + docker: docker-engine=1.13.1-0~ubuntu-xenial + socat: socat=1.7.3.1-1 + + +Files +----- + +A list of files to be written to the host. Files can be given as precise content or extracted from a tarball specified by url: + +.. code-block:: yaml + + - path: /etc/direct-content + content: |- + This + exact + text + - path: /etc/from-tar + tar_url: http://example.com/file + tar_source: dir/file.txt + +Images +------ + +Core Images +^^^^^^^^^^^ + +These images are used for essential functionality: + +``coredns`` + coredns_ is configured and used for Kubernetes API discovery during + bootstrapping. + +``kubectl`` + Used for label application and validation tasks during bootstrapping. + + +Convenience Images +^^^^^^^^^^^^^^^^^^ + +The ``helm`` image is available for convenience. + + +.. _coredns: https://github.com/coredns/coredns + + +Packages +-------- + +Repository Configuration +^^^^^^^^^^^^^^^^^^^^^^^^ + +Additional APT repositories can be configured using the ``repositories`` and +``keys`` fields of the ``SystemPackages`` document: + +``repositories`` + A list of APT source lines to be configured during genesis or join. + +``keys`` + A list of public PGP keys that can be used to verify installed packages. + + +Package Configuration +^^^^^^^^^^^^^^^^^^^^^ + +The ``required`` key specifies packages that are required for all deployments, +and the ``additional`` key allows arbitrary additional system packages to be +installed. The ``additional`` key is particularly useful for installing +packages such as `ceph-common`. diff --git a/docs/source/configuration/index.rst b/docs/source/configuration/index.rst new file mode 100644 index 00000000..ba911518 --- /dev/null +++ b/docs/source/configuration/index.rst @@ -0,0 +1,26 @@ +Configuration +============= + +Promenade is configured using a set of Deckhand_ compatible configuration +documents and a bootstrapping Armada_ manifest that is responsible for +deploying core components into the cluster. + +Details about Promenade-specific documents can be found here: + +.. toctree:: + :maxdepth: 2 + :caption: Documents + + docker + genesis + host-system + kubernetes-network + kubernetes-node + + +The provided Armada_ manifest and will be applied on the genesis node as soon +as it is healthy. + + +.. _Armada: https://github.com/att-comdev/armada +.. _Deckhand: https://github.com/att-comdev/deckhand diff --git a/docs/source/configuration/kubernetes-network.rst b/docs/source/configuration/kubernetes-network.rst new file mode 100644 index 00000000..7c442572 --- /dev/null +++ b/docs/source/configuration/kubernetes-network.rst @@ -0,0 +1,85 @@ +Kubernetes Network +================== + +Configuration for Kubernetes networking during bootstrapping and for the +``kubelet``. + + +Sample Document +--------------- + +.. code-block:: yaml + + schema: promenade/KubernetesNetwork/v1 + metadata: + schema: metadata/Document/v1 + name: kubernetes-network + layeringDefinition: + abstract: false + layer: site + data: + dns: + cluster_domain: cluster.local + service_ip: 10.96.0.10 + bootstrap_validation_checks: + - calico-etcd.kube-system.svc.cluster.local + - kubernetes-etcd.kube-system.svc.cluster.local + - kubernetes.default.svc.cluster.local + upstream_servers: + - 8.8.8.8 + - 8.8.4.4 + + kubernetes: + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + service_ip: 10.96.0.1 + + etcd: + service_ip: 10.96.0.2 + + hosts_entries: + - ip: 192.168.77.1 + names: + - registry + + proxy: + url: http://proxy.example.com:8080 + additional_no_proxy: + - 192.168.77.1 + + +DNS +--- + +The data in the ``dns`` key is used for bootstrapping and ``kubelet`` +configuration of cluster and host-level DNS, which is provided by coredns_. + +``bootstrap_validation_checks`` + Domain names to resolve during the genesis and join processes for validation. + +``cluster_domain`` + The Kubernetes cluster domain. Used by the ``kubelet``. + +``service_ip`` + The IP to use for cluster DNS. Used by the ``kubelet``. + +``upstream_servers`` + Upstream DNS servers to be configured in `/etc/resolv.conf`. + + +Kubernetes +---------- + +The ``kubernetes`` key contains: + +``pod_cidr`` + The CIDR from which the Kubernetes Controller Manager assigns pod IPs. + +``service_cidr`` + The CIDR from which the Kubernetes Controller Manager assigns service IPs. + +``service_ip`` + The in-cluster Kubernetes service IP. + + +.. _coredns: https://github.com/coredns/coredns diff --git a/docs/source/configuration/kubernetes-node.rst b/docs/source/configuration/kubernetes-node.rst new file mode 100644 index 00000000..78e425d0 --- /dev/null +++ b/docs/source/configuration/kubernetes-node.rst @@ -0,0 +1,55 @@ +Kubernetes Node +=============== + +Configuration for a basic node in the cluster. + + +Sample Document +--------------- + +Here is a sample document: + +.. code-block:: yaml + + schema: promenade/KubernetesNode/v1 + metadata: + schema: metadata/Document/v1 + name: n1 + layeringDefinition: + abstract: false + layer: site + data: + hostname: n1 + ip: 192.168.77.11 + join_ip: 192.168.77.10 + labels: + static: + - node-role.kubernetes.io/master= + dynamic: + - calico-etcd=enabled + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - ucp-control-plane=enabled + + +Host Information +---------------- + +Essential host-specific information is specified in this document, including +the ``hostname``, ``ip``, and ``join_ip``. + +The ``join_ip`` is used to specify which host should be used when adding a node +to the cluster. + + +Labels +------ + +Kubernetes labels can be specified under the ``labels`` key in two ways: + +1. Via the ``static`` key, which is a list of labels to be applied immediately + when the ``kubelet`` process starts. +2. Via the ``dynamic`` key, which is a list of labels to be applied after the + node is marked as `Ready` by Kubernetes. diff --git a/docs/source/getting-started.rst b/docs/source/getting-started.rst index 9afd7200..0ed3e44a 100644 --- a/docs/source/getting-started.rst +++ b/docs/source/getting-started.rst @@ -7,74 +7,77 @@ Development Deployment using Vagrant ^^^^^^^^^^^^^^^^^^^^^^^^ +Initial Setup of Vagrant +~~~~~~~~~~~~~~~~~~~~~~~~ + Deployment using Vagrant uses KVM instead of Virtualbox due to better performance of disk and networking, which both have significant impact on the stability of the etcd clusters. Make sure you have [Vagrant](https://vagrantup.com) installed, then -run `./tools/full-vagrant-setup.sh`, which will do the following: +run `./tools/vagrant/full-vagrant-setup.sh`, which will do the following: * Install Vagrant libvirt plugin and its dependencies * Install NFS dependencies for Vagrant volume sharing * Install [packer](https://packer.io) and build a KVM image for Ubuntu 16.04 -Generate the per-host configuration, certificates and keys to be used: +Deployment +~~~~~~~~~~ +A complete set of configuration that works with the `Vagrantfile` in the +top-level directory is provided in the `example` directory. + +To exercise that example, first generate certs and combine the configuration +into usable parts: .. code-block:: bash - mkdir configs - docker run --rm -t -v $(pwd):/target quay.io/attcomdev/promenade:latest promenade -v generate -c /target/example/vagrant-input-config.yaml -o /target/configs - + ./tools/build-example.sh Start the VMs: .. code-block:: bash - vagrant up + vagrant up --parallel -Start the genesis node: +Then bring up the genesis node: .. code-block:: bash - vagrant ssh n0 -c 'sudo bash /vagrant/configs/up.sh /vagrant/configs/n0.yaml' + vagrant ssh n0 -c 'sudo /vagrant/example/scripts/genesis.sh' -Join the master nodes: +Join additional master nodes: .. code-block:: bash - vagrant ssh n1 -c 'sudo bash /vagrant/configs/up.sh /vagrant/configs/n1.yaml' - vagrant ssh n2 -c 'sudo bash /vagrant/configs/up.sh /vagrant/configs/n2.yaml' + vagrant ssh n1 -c 'sudo /vagrant/example/scripts/join-n1.sh' + vagrant ssh n2 -c 'sudo /vagrant/example/scripts/join-n2.sh' -Join the worker node: +Re-provision the genesis node as a normal master: .. code-block:: bash - vagrant ssh n3 -c 'sudo bash /vagrant/configs/up.sh /vagrant/configs/n3.yaml' + vagrant ssh n0 -c 'sudo promenade-teardown' + vagrant ssh n1 -c 'sudo kubectl delete node n0' + vagrant destroy -f n0 + vagrant up n0 + vagrant ssh n0 -c 'sudo /vagrant/example/scripts/join-n0.sh' + +Join the remaining worker: + +.. code-block:: bash + + vagrant ssh n3 -c 'sudo /vagrant/example/scripts/join-n3.sh' + Building the image ^^^^^^^^^^^^^^^^^^ +To build the image directly, you can use the standard Docker build command: + .. code-block:: bash docker build -t promenade:local . - -For development, you may wish to save it and have the `up.sh` script load it: - -.. code-block:: bash - - docker save -o promenade.tar promenade:local - - -Then on a node: - -.. code-block:: bash - - PROMENADE_LOAD_IMAGE=/vagrant/promenade.tar bash /vagrant/up.sh /vagrant/path/to/node-config.yaml - - -These commands are combined in a convenience script at `tools/dev-build.sh`. - To build the image from behind a proxy, you can: .. code-block:: bash @@ -84,8 +87,19 @@ To build the image from behind a proxy, you can: docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$http_proxy --build-arg no_proxy=$no_proxy -t promenade:local . +For convenience, there is a script which builds an image from the current code, +then uses it to construct scripts for the example: + +.. code-block:: bash + + ./tools/dev-build.sh + +*NOTE* the ``dev-build.sh`` script puts Promenade in debug mode, which will +instruct it to use Vagrant's shared directory to source local charts. + + Using Promenade Behind a Proxy ------------------------------ -To use Promenade from behind a proxy, use the proxy settings described in the -[configuration docs](configuration.md). +To use Promenade from behind a proxy, use the proxy settings see +:doc:`configuration/kubernetes-network`. diff --git a/docs/source/index.rst b/docs/source/index.rst index 4ac3306d..c34a681c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -31,4 +31,4 @@ Promenade Configuration Guide :maxdepth: 2 getting-started - configuration + configuration/index diff --git a/example/.gitignore b/example/.gitignore new file mode 100644 index 00000000..cb9919a3 --- /dev/null +++ b/example/.gitignore @@ -0,0 +1,2 @@ +certificates.yaml +scripts diff --git a/example/bootstrap-armada-config.yaml b/example/bootstrap-armada-config.yaml new file mode 100644 index 00000000..19bb3933 --- /dev/null +++ b/example/bootstrap-armada-config.yaml @@ -0,0 +1,958 @@ +--- +schema: armada/Manifest/v1 +metadata: + schema: metadata/Document/v1 + name: cluster-bootstrap + layeringDefinition: + abstract: false + layer: site +data: + release_prefix: ucp + chart_groups: + - kubernetes-proxy + - container-networking + - dns + - kubernetes + - kubernetes-rbac +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-proxy + layeringDefinition: + abstract: false + layer: site +data: + description: Kubernetes proxy + sequenced: true + chart_group: + - kubernetes-proxy +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: container-networking + layeringDefinition: + abstract: false + layer: site +data: + description: Container networking via Calico + sequenced: true + chart_group: + - calico-etcd + - calico +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: dns + layeringDefinition: + abstract: false + layer: site +data: + description: Cluster DNS + chart_group: + - coredns +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-rbac + layeringDefinition: + abstract: false + layer: site +data: + description: Role Based Access Control configuration for Kubernetes + sequenced: true + chart_group: + - kubernetes-rbac +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes + layeringDefinition: + abstract: false + layer: site +data: + description: Kubernetes components + chart_group: + - kubernetes-etcd + - kubernetes-apiserver + - kubernetes-controller-manager + - kubernetes-scheduler +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: helm-toolkit +data: + chart_name: helm-toolkit + release: helm-toolkit + namespace: helm-toolkit + timeout: 600 + upgrade: + no_hooks: true + values: {} + source: + type: git + location: https://git.openstack.org/openstack/openstack-helm + subpath: helm-toolkit + reference: master + dependencies: [] +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-proxy + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: $ + dest: + path: '$.values.tls.ca' + - + src: + schema: deckhand/Certificate/v1 + name: proxy + path: $ + dest: + path: '$.values.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: proxy + path: $ + dest: + path: '$.values.tls.key' +data: + chart_name: proxy + release: kubernetes-proxy + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + tls: + ca: placeholder + cert: placeholder + key: placeholder + images: + proxy: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + network: + kubernetes_netloc: apiserver.kubernetes.promenade:6443 + pod_cidr: 10.97.0.0/16 + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: proxy + dependencies: [] +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: calico-etcd + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: calico-etcd + path: $ + dest: + path: '$.values.tls.client.ca' + - + src: + schema: deckhand/CertificateAuthority/v1 + name: calico-etcd-peer + path: $ + dest: + path: '$.values.tls.peer.ca' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-anchor + path: $ + dest: + path: '$.values.anchor.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-anchor + path: $ + dest: + path: '$.values.anchor.tls.key' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n0 + path: $ + dest: + path: '$.values.nodes[0].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n0 + path: $ + dest: + path: '$.values.nodes[0].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n0-peer + path: $ + dest: + path: '$.values.nodes[0].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n0-peer + path: $ + dest: + path: '$.values.nodes[0].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n1 + path: $ + dest: + path: '$.values.nodes[1].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n1 + path: $ + dest: + path: '$.values.nodes[1].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n1-peer + path: $ + dest: + path: '$.values.nodes[1].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n1-peer + path: $ + dest: + path: '$.values.nodes[1].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n2 + path: $ + dest: + path: '$.values.nodes[2].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n2 + path: $ + dest: + path: '$.values.nodes[2].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n2-peer + path: $ + dest: + path: '$.values.nodes[2].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n2-peer + path: $ + dest: + path: '$.values.nodes[2].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n3 + path: $ + dest: + path: '$.values.nodes[3].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n3 + path: $ + dest: + path: '$.values.nodes[3].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-n3-peer + path: $ + dest: + path: '$.values.nodes[3].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-n3-peer + path: $ + dest: + path: '$.values.nodes[3].tls.peer.key' + +data: + chart_name: etcd + release: calico-etcd + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + anchor: + etcdctl_endpoint: 10.96.232.136 + node_selector: + key: calico-etcd + value: enabled + tls: + cert: placeholder + key: placeholder + etcd: + host_data_path: /var/lib/etcd/calico + host_etc_path: /etc/etcd/calico + bootstrapping: + enabled: true + host_directory: /var/lib/anchor + filename: calico-etcd-bootstrap + images: + etcd: quay.io/coreos/etcd:v3.0.17 + etcdctl: quay.io/coreos/etcd:v3.0.17 + nodes: + - name: n0 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + - name: n1 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + - name: n2 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + # n3 is here to demonstrate movability of the cluster + - name: n3 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + service: + name: calico-etcd + ip: 10.96.232.136 + client: + port: 6666 + target_port: 6666 + peer: + port: 6667 + target_port: 6667 + tls: + client: + ca: placeholder + peer: + ca: placeholder + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: etcd + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: calico + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: calico-etcd + path: $ + dest: + path: '$.values.etcd.tls.ca' + - + src: + schema: deckhand/Certificate/v1 + name: calico-node + path: $ + dest: + path: '$.values.etcd.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-node + path: $ + dest: + path: '$.values.etcd.tls.key' +data: + chart_name: calico + release: calico + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + calico: + ip_autodetection_method: interface=ens6 + pod_ip_cidr: 10.97.0.0/16 + ctl: + install_on_host: true + etcd: + service: + ip: 10.96.232.136 + port: 6666 + tls: + ca: placeholder + cert: placeholder + key: placeholder + images: + cni: quay.io/calico/cni:v1.11.0 + ctl: quay.io/calico/ctl:v1.6.1 + node: quay.io/calico/node:v2.6.1 + policy_controller: quay.io/calico/kube-controllers:v1.0.0 + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: calico + dependencies: [] +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: coredns + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: $ + dest: + path: '$.values.tls.ca' + - + src: + schema: deckhand/Certificate/v1 + name: coredns + path: $ + dest: + path: '$.values.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: coredns + path: $ + dest: + path: '$.values.tls.key' +data: + chart_name: coredns + release: coredns + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + coredns: + cluster_domain: cluster.local + upstream_nameservers: + - 8.8.8.8 + - 8.8.4.4 + + zones: + - name: promenade + services: + - bind_name: apiserver.kubernetes + service: + name: kubernetes-apiserver + namespace: kube-system + - bind_name: etcd.kubernetes + service: + name: kubernetes-etcd + namespace: kube-system + - bind_name: etcd.calico + service: + name: calico-etcd + namespace: kube-system + images: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + coredns: coredns/coredns:011 + tls: + ca: placeholder + cert: placeholder + key: placeholder + network: + kubernetes_netloc: apiserver.kubernetes.promenade:6443 + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: coredns + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-apiserver + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: $ + dest: + path: $.values.tls.ca + + - + src: + schema: deckhand/Certificate/v1 + name: apiserver + path: $ + dest: + path: $.values.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: apiserver + path: $ + dest: + path: $.values.tls.key + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes-etcd + path: $ + dest: + path: $.values.apiserver.etcd.tls.ca + - + src: + schema: deckhand/Certificate/v1 + name: apiserver-etcd + path: $ + dest: + path: $.values.apiserver.etcd.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: apiserver-etcd + path: $ + dest: + path: $.values.apiserver.etcd.tls.key + - + src: + schema: deckhand/PublicKey/v1 + name: service-account + path: $ + dest: + path: $.values.service_account.public_key + +data: + chart_name: apiserver + release: kubernetes-apiserver + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + apiserver: + etcd: + endpoints: https://etcd.kubernetes.promenade:2379 + tls: + ca: placeholder + cert: placeholder + key: placeholder + images: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + apiserver: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + service_account: + public_key: placeholder + tls: + ca: placeholder + cert: placeholder + key: placeholder + network: + kubernetes_service_ip: 10.96.0.1 + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: apiserver + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-controller-manager + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: $ + dest: + path: $.values.tls.ca + + - + src: + schema: deckhand/Certificate/v1 + name: controller-manager + path: $ + dest: + path: $.values.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: controller-manager + path: $ + dest: + path: $.values.tls.key + - + src: + schema: deckhand/PrivateKey/v1 + name: service-account + path: $ + dest: + path: $.values.service_account.private_key + +data: + chart_name: controller_manager + release: kubernetes-controller-manager + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + images: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + controller_manager: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + service_account: + private_key: placeholder + tls: + ca: placeholder + cert: placeholder + key: placeholder + network: + kubernetes_netloc: apiserver.kubernetes.promenade:6443 + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: controller_manager + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-scheduler + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: $ + dest: + path: $.values.tls.ca + - + src: + schema: deckhand/Certificate/v1 + name: scheduler + path: $ + dest: + path: $.values.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: scheduler + path: $ + dest: + path: $.values.tls.key + +data: + chart_name: scheduler + release: kubernetes-scheduler + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + tls: + ca: placeholder + cert: placeholder + key: placeholder + + network: + kubernetes_netloc: apiserver.kubernetes.promenade:6443 + + images: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + scheduler: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: scheduler + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-etcd + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes-etcd + path: $ + dest: + path: '$.values.tls.client.ca' + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes-etcd-peer + path: $ + dest: + path: '$.values.tls.peer.ca' + + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-anchor + path: $ + dest: + path: '$.values.anchor.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-anchor + path: $ + dest: + path: '$.values.anchor.tls.key' + + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-n0 + path: $ + dest: + path: '$.values.nodes[0].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-n0 + path: $ + dest: + path: '$.values.nodes[0].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-n0-peer + path: $ + dest: + path: '$.values.nodes[0].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-n0-peer + path: $ + dest: + path: '$.values.nodes[0].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-n1 + path: $ + dest: + path: '$.values.nodes[1].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-n1 + path: $ + dest: + path: '$.values.nodes[1].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-n1-peer + path: $ + dest: + path: '$.values.nodes[1].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-n1-peer + path: $ + dest: + path: '$.values.nodes[1].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-n2 + path: $ + dest: + path: '$.values.nodes[2].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-n2 + path: $ + dest: + path: '$.values.nodes[2].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-n2-peer + path: $ + dest: + path: '$.values.nodes[2].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-n2-peer + path: $ + dest: + path: '$.values.nodes[2].tls.peer.key' + +data: + chart_name: etcd + release: kubernetes-etcd + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + anchor: + etcdctl_endpoint: 10.96.0.2 + node_selector: + key: kubernetes-etcd + value: enabled + tls: + cert: placeholder + key: placeholder + etcd: + host_data_path: /var/lib/etcd/kubernetes + host_etc_path: /etc/etcd/kubernetes + images: + etcd: quay.io/coreos/etcd:v3.0.17 + etcdctl: quay.io/coreos/etcd:v3.0.17 + nodes: + - name: n0 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + - name: n1 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + - name: n2 + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + service: + name: kubernetes-etcd + ip: 10.96.0.2 + client: + port: 2379 + target_port: 2379 + peer: + port: 2380 + target_port: 2380 + tls: + client: + ca: placeholder + peer: + ca: placeholder + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: etcd + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-rbac + layeringDefinition: + abstract: false + layer: site +data: + chart_name: rbac + release: rbac + namespace: kube-system + timeout: 600 + values: {} + upgrade: + no_hooks: true + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: rbac + dependencies: [] +... diff --git a/example/gen-ca/ca-csr.json b/example/gen-ca/ca-csr.json deleted file mode 100644 index 6bff70f7..00000000 --- a/example/gen-ca/ca-csr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "CN": "Kubernetes", - "key": { - "algo": "rsa", - "size": 4096 - }, - "names": [ - { - "C": "US", - "L": "Saint Louis", - "O": "Kubernetes", - "OU": "CA", - "ST": "Missouri" - } - ] -} diff --git a/example/genesis-config.yaml b/example/genesis-config.yaml new file mode 100644 index 00000000..0bd3f808 --- /dev/null +++ b/example/genesis-config.yaml @@ -0,0 +1,36 @@ +--- +schema: promenade/Genesis/v1 +metadata: + schema: metadata/Document/v1 + name: genesis + layeringDefinition: + abstract: false + layer: site +data: + hostname: n0 + ip: 192.168.77.10 + labels: + static: + - calico-etcd=enabled + - node-role.kubernetes.io/master= + dynamic: + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - promenade-genesis=enabled + - ucp-control-plane=enabled + images: + armada: quay.io/attcomdev/armada:latest + helm: + tiller: gcr.io/kubernetes-helm/tiller:v2.5.1 + kubernetes: + apiserver: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + controller-manager: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + etcd: quay.io/coreos/etcd:v3.0.17 + scheduler: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + files: + - path: /var/lib/anchor/calico-etcd-bootstrap + content: "" + mode: 0644 +... diff --git a/example/joining-host-config.yaml b/example/joining-host-config.yaml new file mode 100644 index 00000000..c40b6cfd --- /dev/null +++ b/example/joining-host-config.yaml @@ -0,0 +1,82 @@ +--- +schema: promenade/KubernetesNode/v1 +metadata: + schema: metadata/Document/v1 + name: n0 + layeringDefinition: + abstract: false + layer: site +data: + hostname: n0 + ip: 192.168.77.10 + join_ip: 192.168.77.11 + labels: + static: + - node-role.kubernetes.io/master= + dynamic: + - calico-etcd=enabled + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - ucp-control-plane=enabled +--- +schema: promenade/KubernetesNode/v1 +metadata: + schema: metadata/Document/v1 + name: n1 + layeringDefinition: + abstract: false + layer: site +data: + hostname: n1 + ip: 192.168.77.11 + join_ip: 192.168.77.10 + labels: + static: + - node-role.kubernetes.io/master= + dynamic: + - calico-etcd=enabled + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - ucp-control-plane=enabled +--- +schema: promenade/KubernetesNode/v1 +metadata: + schema: metadata/Document/v1 + name: n2 + layeringDefinition: + abstract: false + layer: site +data: + hostname: n2 + ip: 192.168.77.12 + join_ip: 192.168.77.10 + labels: + static: + - node-role.kubernetes.io/master= + dynamic: + - calico-etcd=enabled + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - ucp-control-plane=enabled +--- +schema: promenade/KubernetesNode/v1 +metadata: + schema: metadata/Document/v1 + name: n3 + layeringDefinition: + abstract: false + layer: site +data: + hostname: n3 + ip: 192.168.77.13 + join_ip: 192.168.77.11 + labels: + dynamic: + - ucp-control-plane=enabled +... diff --git a/example/site-config.yaml b/example/site-config.yaml new file mode 100644 index 00000000..0a248e9f --- /dev/null +++ b/example/site-config.yaml @@ -0,0 +1,112 @@ +--- +schema: promenade/KubernetesNetwork/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-network + layeringDefinition: + abstract: false + layer: site +data: + dns: + cluster_domain: cluster.local + service_ip: 10.96.0.10 + bootstrap_validation_checks: + - calico-etcd.kube-system.svc.cluster.local + - kubernetes-etcd.kube-system.svc.cluster.local + - kubernetes.default.svc.cluster.local + upstream_servers: + - 8.8.8.8 + - 8.8.4.4 + + kubernetes: + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + service_ip: 10.96.0.1 + + etcd: + service_ip: 10.96.0.2 + + hosts_entries: + - ip: 192.168.77.1 + names: + - registry + +# proxy: +# url: http://proxy.example.com:8080 +# additional_no_proxy: +# - 10.0.1.1 +--- +schema: promenade/Docker/v1 +metadata: + schema: metadata/Document/v1 + name: docker + layeringDefinition: + abstract: false + layer: site +data: + config: + insecure-registries: + - registry:5000 + live-restore: true + storage-driver: overlay2 +--- +schema: promenade/HostSystem/v1 +metadata: + schema: metadata/Document/v1 + name: host-system + layeringDefinition: + abstract: false + layer: site +data: + files: + - path: /opt/kubernetes/bin/kubelet + tar_url: https://dl.k8s.io/v1.8.0/kubernetes-node-linux-amd64.tar.gz + tar_path: kubernetes/node/bin/kubelet + mode: 0555 + images: + coredns: coredns/coredns:011 + helm: + helm: lachlanevenson/k8s-helm:v2.5.1 + kubernetes: + kubectl: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + packages: + repositories: + - deb http://apt.dockerproject.org/repo ubuntu-xenial main + keys: + - |- + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFWln24BEADrBl5p99uKh8+rpvqJ48u4eTtjeXAWbslJotmC/CakbNSqOb9o + ddfzRvGVeJVERt/Q/mlvEqgnyTQy+e6oEYN2Y2kqXceUhXagThnqCoxcEJ3+KM4R + mYdoe/BJ/J/6rHOjq7Omk24z2qB3RU1uAv57iY5VGw5p45uZB4C4pNNsBJXoCvPn + TGAs/7IrekFZDDgVraPx/hdiwopQ8NltSfZCyu/jPpWFK28TR8yfVlzYFwibj5WK + dHM7ZTqlA1tHIG+agyPf3Rae0jPMsHR6q+arXVwMccyOi+ULU0z8mHUJ3iEMIrpT + X+80KaN/ZjibfsBOCjcfiJSB/acn4nxQQgNZigna32velafhQivsNREFeJpzENiG + HOoyC6qVeOgKrRiKxzymj0FIMLru/iFF5pSWcBQB7PYlt8J0G80lAcPr6VCiN+4c + NKv03SdvA69dCOj79PuO9IIvQsJXsSq96HB+TeEmmL+xSdpGtGdCJHHM1fDeCqkZ + hT+RtBGQL2SEdWjxbF43oQopocT8cHvyX6Zaltn0svoGs+wX3Z/H6/8P5anog43U + 65c0A+64Jj00rNDr8j31izhtQMRo892kGeQAaaxg4Pz6HnS7hRC+cOMHUU4HA7iM + zHrouAdYeTZeZEQOA7SxtCME9ZnGwe2grxPXh/U/80WJGkzLFNcTKdv+rwARAQAB + tDdEb2NrZXIgUmVsZWFzZSBUb29sIChyZWxlYXNlZG9ja2VyKSA8ZG9ja2VyQGRv + Y2tlci5jb20+iQI4BBMBAgAiBQJVpZ9uAhsvBgsJCAcDAgYVCAIJCgsEFgIDAQIe + AQIXgAAKCRD3YiFXLFJgnbRfEAC9Uai7Rv20QIDlDogRzd+Vebg4ahyoUdj0CH+n + Ak40RIoq6G26u1e+sdgjpCa8jF6vrx+smpgd1HeJdmpahUX0XN3X9f9qU9oj9A4I + 1WDalRWJh+tP5WNv2ySy6AwcP9QnjuBMRTnTK27pk1sEMg9oJHK5p+ts8hlSC4Sl + uyMKH5NMVy9c+A9yqq9NF6M6d6/ehKfBFFLG9BX+XLBATvf1ZemGVHQusCQebTGv + 0C0V9yqtdPdRWVIEhHxyNHATaVYOafTj/EF0lDxLl6zDT6trRV5n9F1VCEh4Aal8 + L5MxVPcIZVO7NHT2EkQgn8CvWjV3oKl2GopZF8V4XdJRl90U/WDv/6cmfI08GkzD + YBHhS8ULWRFwGKobsSTyIvnbk4NtKdnTGyTJCQ8+6i52s+C54PiNgfj2ieNn6oOR + 7d+bNCcG1CdOYY+ZXVOcsjl73UYvtJrO0Rl/NpYERkZ5d/tzw4jZ6FCXgggA/Zxc + jk6Y1ZvIm8Mt8wLRFH9Nww+FVsCtaCXJLP8DlJLASMD9rl5QS9Ku3u7ZNrr5HWXP + HXITX660jglyshch6CWeiUATqjIAzkEQom/kEnOrvJAtkypRJ59vYQOedZ1sFVEL + MXg2UCkD/FwojfnVtjzYaTCeGwFQeqzHmM241iuOmBYPeyTY5veF49aBJA1gEJOQ + TvBR8Q== + =Fm3p + -----END PGP PUBLIC KEY BLOCK----- + additional: + - curl + - jq + required: + docker: docker-engine=1.13.1-0~ubuntu-xenial + socat: socat=1.7.3.1-1 +... diff --git a/example/vagrant-input-config.yaml b/example/vagrant-input-config.yaml deleted file mode 100644 index 31e7f67c..00000000 --- a/example/vagrant-input-config.yaml +++ /dev/null @@ -1,123 +0,0 @@ ---- -apiVersion: promenade/v1 -kind: Cluster -metadata: - name: example - target: none -spec: - nodes: - n0: - ip: 192.168.77.10 - roles: - - master - - genesis - additional_labels: - - beta.kubernetes.io/arch=amd64 - n1: - ip: 192.168.77.11 - roles: - - master - additional_labels: - - beta.kubernetes.io/arch=amd64 - n2: - ip: 192.168.77.12 - roles: - - master - additional_labels: - - beta.kubernetes.io/arch=amd64 - n3: - ip: 192.168.77.13 - roles: - - worker - additional_labels: - - beta.kubernetes.io/arch=amd64 ---- -apiVersion: promenade/v1 -kind: Network -metadata: - cluster: example - name: example - target: all -spec: - cluster_domain: cluster.local - cluster_dns: 10.96.0.10 - kube_service_ip: 10.96.0.1 - pod_ip_cidr: 10.97.0.0/16 - service_ip_cidr: 10.96.0.0/16 - calico_etcd_service_ip: 10.96.232.136 - calico_interface: enp0s8 - dns_servers: - - 8.8.8.8 - - 8.8.4.4 - #http_proxy: http://proxy.example.com:8080 - #https_proxy: https://proxy.example.com:8080 ---- -apiVersion: promenade/v1 -kind: Versions -metadata: - cluster: example - name: example - target: all -spec: - images: - armada: quay.io/attcomdev/armada:latest - calico: - cni: quay.io/calico/cni:v1.9.1 - etcd: quay.io/coreos/etcd:v3.2.1 - node: quay.io/calico/node:v1.3.0 - policy-controller: quay.io/calico/kube-policy-controller:v0.6.0 - kubernetes: - apiserver: gcr.io/google_containers/hyperkube-amd64:v1.6.8 - controller-manager: quay.io/attcomdev/kube-controller-manager:v1.6.8 - dns: - dnsmasq: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.2 - kubedns: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.2 - sidecar: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.2 - etcd: quay.io/coreos/etcd:v3.2.1 - kubectl: gcr.io/google_containers/hyperkube-amd64:v1.6.8 - proxy: gcr.io/google_containers/hyperkube-amd64:v1.6.8 - scheduler: gcr.io/google_containers/hyperkube-amd64:v1.6.8 - promenade: quay.io/attcomdev/promenade:latest - tiller: gcr.io/kubernetes-helm/tiller:v2.5.0 - packages: - docker: docker.io=1.12.6-0ubuntu1~16.04.1 - dnsmasq: dnsmasq=2.75-1ubuntu0.16.04.2 - socat: socat=1.7.3.1-1 - additional_packages: - - ceph-common=10.2.7-0ubuntu0.16.04.1 ---- -schema: armada/Manifest/v1 -metadata: - schema: metadata/Document/v1 - name: example-application -data: - release_prefix: example - chart_groups: - - example-application ---- -schema: armada/ChartGroup/v1 -metadata: - schema: metadata/Document/v1 - name: example-application -data: - description: Just an example - chart_group: - - redis ---- -schema: armada/Chart/v1 -metadata: - schema: metadata/Document/v1 - name: redis -data: - chart_name: redis - release: example-redis - namespace: default - timeout: 600 - values: - persistence: - enabled: false - source: - type: git - location: https://github.com/kubernetes/charts.git - subpath: stable/redis - dependencies: [] diff --git a/promenade/builder.py b/promenade/builder.py new file mode 100644 index 00000000..3453f95a --- /dev/null +++ b/promenade/builder.py @@ -0,0 +1,119 @@ +from . import logging, renderer +import io +import itertools +import os +import requests +import tarfile + +__all__ = ['Builder'] + +LOG = logging.getLogger(__name__) + + +class Builder: + def __init__(self, config, *, validators=False): + self.config = config + self.validators = validators + self._file_cache = None + + @property + def file_cache(self): + if not self._file_cache: + self._build_file_cache() + return self._file_cache + + def _build_file_cache(self): + self._file_cache = {} + for file_spec in self._file_specs: + path = file_spec['path'] + if 'content' in file_spec: + data = file_spec['content'] + elif 'tar_url' in file_spec: + data = _fetch_tar_content( + url=file_spec['tar_url'], path=file_spec['tar_path']) + self._file_cache[path] = { + 'path': path, + 'data': data, + 'mode': file_spec['mode'], + } + + @property + def _file_specs(self): + return itertools.chain( + self.config.get_path('HostSystem:files', []), + self.config.get_path('Genesis:files', [])) + + def build_all(self, *, output_dir): + self.build_genesis(output_dir=output_dir) + for node_document in self.config.iterate( + schema='promenade/KubernetesNode/v1'): + self.build_node(node_document, output_dir=output_dir) + + if self.validators: + validate_script = renderer.render_template( + self.config, template='scripts/validate-cluster.sh') + _write_script(output_dir, 'validate-cluster.sh', validate_script) + + def build_genesis(self, *, output_dir): + LOG.info('Building genesis script') + sub_config = self.config.extract_genesis_config() + tarball = renderer.build_tarball_from_roles( + config=sub_config, + roles=['common', 'genesis'], + file_specs=self.file_cache.values()) + + script = renderer.render_template( + sub_config, + template='scripts/genesis.sh', + context={'tarball': tarball}) + + _write_script(output_dir, 'genesis.sh', script) + + if self.validators: + validate_script = renderer.render_template( + sub_config, template='scripts/validate-genesis.sh') + _write_script(output_dir, 'validate-genesis.sh', validate_script) + + def build_node(self, node_document, *, output_dir): + node_name = node_document['metadata']['name'] + LOG.info('Building script for node %s', node_name) + sub_config = self.config.extract_node_config(node_name) + file_spec_paths = [ + f['path'] for f in self.config.get_path('HostSystem:files', []) + ] + file_specs = [self.file_cache[p] for p in file_spec_paths] + tarball = renderer.build_tarball_from_roles( + config=sub_config, roles=['common', 'join'], file_specs=file_specs) + + script = renderer.render_template( + sub_config, + template='scripts/join.sh', + context={'tarball': tarball}) + + _write_script(output_dir, _join_name(node_name), script) + + if self.validators: + validate_script = renderer.render_template( + sub_config, template='scripts/validate-join.sh') + _write_script(output_dir, 'validate-%s.sh' % node_name, + validate_script) + + +def _fetch_tar_content(*, url, path): + response = requests.get(url) + response.raise_for_status() + f = io.BytesIO(response.content) + tf = tarfile.open(fileobj=f, mode='r') + buf_reader = tf.extractfile(path) + return buf_reader.read() + + +def _join_name(node_name): + return 'join-%s.sh' % node_name + + +def _write_script(output_dir, name, script): + path = os.path.join(output_dir, name) + with open(path, 'w') as f: + os.fchmod(f.fileno(), 0o555) + f.write(script) diff --git a/promenade/cli.py b/promenade/cli.py index ddd15bd6..8dea4bcf 100644 --- a/promenade/cli.py +++ b/promenade/cli.py @@ -1,48 +1,69 @@ -from . import generator, logging, operator +from . import builder, config, exceptions, generator, logging import click +import os +import sys __all__ = [] - LOG = logging.getLogger(__name__) @click.group() @click.option('-v', '--verbose', is_flag=True) def promenade(*, verbose): + if _debug(): + verbose = True logging.setup(verbose=verbose) -@promenade.command(help='Initialize a new cluster on one node') -@click.option('-a', '--asset-dir', default='/assets', - type=click.Path(exists=True, file_okay=False, - dir_okay=True, resolve_path=True), - help='Source path for binaries to deploy.') -@click.option('-c', '--config-path', type=click.File(), - help='Location of cluster configuration data.') -@click.option('--hostname', help='Current hostname.') -@click.option('-t', '--target-dir', default='/target', - type=click.Path(exists=True, file_okay=False, - dir_okay=True, resolve_path=True), - help='Location where templated files will be placed.') -def up(*, asset_dir, config_path, hostname, target_dir): - - op = operator.Operator.from_config(config_path=config_path, - hostname=hostname, - target_dir=target_dir) - - op.up(asset_dir=asset_dir) +@promenade.command('build-all', help='Construct all scripts') +@click.option( + '-o', + '--output-dir', + default='.', + type=click.Path( + exists=True, file_okay=False, dir_okay=True, resolve_path=True), + required=True, + help='Location to write complete cluster configuration.') +@click.option('--validators', is_flag=True, help='Generate validation scripts') +@click.argument('config_files', nargs=-1, type=click.File('rb')) +def build_all(*, config_files, output_dir, validators): + debug = _debug() + try: + c = config.Configuration.from_streams( + debug=debug, streams=config_files) + b = builder.Builder(c, validators=validators) + b.build_all(output_dir=output_dir) + except exceptions.PromenadeException as e: + e.display(debug=debug) + sys.exit(e.EXIT_CODE) -@promenade.command(help='Generate certs and keys') -@click.option('-c', '--config-path', type=click.File(), - required=True, - help='Location of cluster configuration data.') -@click.option('-o', '--output-dir', default='.', - type=click.Path(exists=True, file_okay=False, dir_okay=True, - resolve_path=True), - required=True, - help='Location to write complete cluster configuration.') -def generate(*, config_path, output_dir): - g = generator.Generator.from_config(config_path=config_path) - g.generate_all(output_dir) +@promenade.command('generate-certs', help='Generate a certs for a site') +@click.option( + '-o', + '--output-dir', + type=click.Path( + exists=True, file_okay=False, dir_okay=True, resolve_path=True), + required=True, + help='Location to write *-certificates.yaml') +@click.argument('config_files', nargs=-1, type=click.File('rb')) +@click.option( + '--calico-etcd-service-ip', + default='10.96.232.136', + help='Service IP for calico etcd') +def genereate_certs(*, calico_etcd_service_ip, config_files, output_dir): + debug = _debug() + try: + c = config.Configuration.from_streams( + debug=debug, streams=config_files, substitute=False) + g = generator.Generator( + c, calico_etcd_service_ip=calico_etcd_service_ip) + g.generate(output_dir) + except exceptions.PromenadeException as e: + e.display(debug=debug) + sys.exit(e.EXIT_CODE) + + +def _debug(): + return os.environ.get('PROMENADE_DEBUG', '').lower() in {'1', 'True'} diff --git a/promenade/config.py b/promenade/config.py index e7e927ac..eed64b77 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -1,150 +1,206 @@ -from . import logging -from operator import attrgetter, itemgetter -import itertools +from . import exceptions, logging, validation +import copy +import jinja2 +import jsonpath_ng import yaml -__all__ = ['Configuration', 'Document', 'load'] - +__all__ = ['Configuration'] LOG = logging.getLogger(__name__) -def load(f): - return Configuration(list(map(instantiate_document, yaml.safe_load_all(f)))) - - -def instantiate_document(data): - if data.get('schema', '').startswith('armada'): - return Document({ - 'apiVersion': 'promenade/v1', - 'kind': 'ArmadaDocument', - 'metadata': { - 'name': data['schema'] + '/' + data['metadata']['name'], - 'target': 'none', - }, - 'spec': data, - }) - else: - return Document(data) - - -class Document: - KEYS = { - 'apiVersion', - 'metadata', - 'kind', - 'spec', - } - - SUPPORTED_KINDS = { - 'ArmadaDocument', - - 'Certificate', - 'CertificateAuthority', - 'CertificateAuthorityKey', - 'CertificateKey', - 'Cluster', - 'Etcd', - 'Masters', - 'Network', - 'Node', - 'PrivateKey', - 'PublicKey', - 'Versions', - } - - def __init__(self, data): - if set(data.keys()) != self.KEYS: - LOG.error('data.keys()=%s expected %s', data.keys(), self.KEYS) - raise AssertionError('Did not get expected keys') - assert data['apiVersion'] == 'promenade/v1' - assert data['kind'] in self.SUPPORTED_KINDS - assert data['metadata']['name'] - - self.data = data - - @property - def kind(self): - return self.data['kind'] - - @property - def name(self): - return self.metadata['name'] - - @property - def alias(self): - return self.metadata.get('alias') - - @property - def target(self): - return self.metadata.get('target') - - @property - def metadata(self): - return self.data['metadata'] - - def __getitem__(self, key): - return self.data['spec'][key] - - def get(self, key, default=None): - return self.data['spec'].get(key, default) - - class Configuration: - def __init__(self, documents): - self.documents = sorted(documents, key=attrgetter('kind', 'target')) + def __init__(self, *, documents, debug=False, substitute=True): + if substitute: + documents = _substitute(documents) + self.debug = debug + self.documents = documents - self.validate() + @classmethod + def from_streams(cls, *, streams, **kwargs): + documents = [] + for stream in streams: + stream_name = getattr(stream, 'name') + if stream_name is not None: + LOG.info('Loading documents from %s', stream_name) + stream_documents = list(yaml.safe_load_all(stream)) + validation.check_schemas(stream_documents) + if stream_name is not None: + LOG.info('Successfully validated documents from %s', + stream_name) + documents.extend(stream_documents) + + return cls(documents=documents, **kwargs) + + def __getitem__(self, path): + value = self.get_path(path) + if value: + return value + else: + return jinja2.StrictUndefined('No match found for path %s' % path) + + def get_first(self, *paths): + result = self._get_first(*paths) + if result: + return result + else: + return jinja2.StrictUndefined( + 'Nothing found matching paths: %s' % ','.join(paths)) + + def get(self, *, kind=None, name=None, schema=None): + result = _get(self.documents, kind=kind, schema=schema, name=name) + + if result: + return result['data'] + else: + return jinja2.StrictUndefined( + 'No document found matching kind=%s schema=%s name=%s' % + (kind, schema, name)) + + def iterate(self, *, kind=None, schema=None, labels=None): + if kind is not None: + assert schema is None + schema = 'promenade/%s/v1' % kind - def validate(self): - identifiers = set() for document in self.documents: - identifier = (document.kind, document.name) - if identifier in identifiers: - LOG.error('Found duplicate document in config: kind=%s name=%s', - document.kind, document.name) - raise RuntimeError('Duplicate document') + if _matches_filter(document, schema=schema, labels=labels): + yield document + + def extract_genesis_config(self): + LOG.debug('Extracting genesis config.') + documents = [] + for document in self.documents: + if document['schema'] != 'promenade/KubernetesNode/v1': + documents.append(document) else: - identifiers.add(identifier) + LOG.debug('Excluding schema=%s metadata.name=%s', + document['schema'], _mg(document, 'name')) + return Configuration( + debug=self.debug, documents=documents, substitute=False) - def __getitem__(self, key): - results = [d for d in self.documents if d.kind == key] - if len(results) < 1: - raise KeyError - elif len(results) > 1: - raise KeyError('Too many results.') - else: - return results[0] - - def get(self, *, kind, alias=None, name=None): + def extract_node_config(self, name): + LOG.debug('Extracting node config for %s.', name) + documents = [] for document in self.documents: - if (document.kind == kind - and (not alias or document.alias == alias) - and (not name or document.name == name)) : - return document + schema = document['schema'] + if schema == 'promenade/Genesis/v1': + LOG.debug('Excluding schema=%s metadata.name=%s', schema, + _mg(document, 'name')) + continue + elif schema == 'promenade/KubernetesNode/v1' and _mg( + document, 'name') != name: + LOG.debug('Excluding schema=%s metadata.name=%s', schema, + _mg(document, 'name')) + continue + else: + documents.append(document) + return Configuration( + debug=self.debug, documents=documents, substitute=False) - def iterate(self, *, kind=None, target=None): - if target: - docs = self._iterate_with_target(target) - else: - docs = self.documents + @property + def kubelet_name(self): + for document in self.iterate(kind='Genesis'): + return 'genesis' - for document in docs: - if not kind or document.kind == kind: - yield document + for document in self.iterate(kind='KubernetesNode'): + return document['data']['hostname'] - def get_armada_documents(self): - return [d.data['spec'] for d in self.iterate(kind='ArmadaDocument')] + return jinja2.StrictUndefined( + 'No Genesis or KubernetesNode found while getting kubelet name') - def _iterate_with_target(self, target): - for document in self.documents: - if document.target == target or document.target == 'all': - yield document + def _get_first(self, *paths): + for path in paths: + value = self.get_path(path) + if value: + return value - def write(self, path): - with open(path, 'w') as f: - yaml.dump_all(map(attrgetter('data'), self.documents), - default_flow_style=False, - explicit_start=True, - indent=2, - stream=f) + def get_path(self, path, default=None): + kind, jsonpath = path.split(':') + document = _get(self.documents, kind=kind) + if document: + data = _extract(document['data'], jsonpath) + if data: + return data + return default + + +def _matches_filter(document, *, schema, labels): + matches = True + if schema is not None and not document.get('schema', + '').startswith(schema): + matches = False + + if labels is not None: + document_labels = _mg(document, 'labels', []) + for key, value in labels.items(): + if key not in document_labels: + matches = False + else: + if document_labels[key] != value: + matches = False + + return matches + + +def _get(documents, kind=None, schema=None, name=None): + if kind is not None: + assert schema is None + schema = 'promenade/%s/v1' % kind + + for document in documents: + if (schema == document.get('schema') + and (name is None or name == _mg(document, 'name'))): + return document + + +def _substitute(documents): + result = [] + + for document in documents: + dest_schema = document.get('schema') + dest_name = _mg(document, 'name') + LOG.debug('Checking for substitutions in schema=%s metadata.name=%s', + dest_schema, dest_name) + final_doc = copy.deepcopy(document) + for substitution in _mg(document, 'substitutions', []): + source_schema = substitution['src']['schema'] + source_name = substitution['src']['name'] + source_path = substitution['src']['path'] + dest_path = substitution['dest']['path'] + LOG.debug('Substituting from schema=%s name=%s src_path=%s ' + 'into dest_path=%s', source_schema, source_name, + source_path, dest_path) + source_document = _get( + documents, schema=source_schema, name=source_name) + if source_document is None: + msg = 'Failed to find source document for subsitution. ' \ + 'dest_schema=%s dest_name=%s ' \ + 'source_schema=%s source_name=%s' \ + % (dest_schema, dest_name, source_schema, source_name) + LOG.critical(msg) + raise exceptions.ValidationException(msg) + + source_value = _extract(source_document['data'], + substitution['src']['path']) + final_doc['data'] = _replace(final_doc['data'], source_value, + substitution['dest']['path']) + + result.append(final_doc) + + return result + + +def _extract(document, jsonpath): + p = jsonpath_ng.parse(jsonpath) + matches = p.find(document) + if matches: + return matches[0].value + + +def _replace(document, value, jsonpath): + p = jsonpath_ng.parse(jsonpath) + return p.update(document, value) + + +def _mg(document, field, default=None): + return document.get('metadata', {}).get(field, default) diff --git a/promenade/exceptions.py b/promenade/exceptions.py new file mode 100644 index 00000000..edfd7827 --- /dev/null +++ b/promenade/exceptions.py @@ -0,0 +1,21 @@ +import logging + +LOG = logging.getLogger(__name__) + + +class PromenadeException(Exception): + EXIT_CODE = 1 + + def __init__(self, message, *, trace=True): + self.message = message + self.trace = trace + + def display(self, debug=False): + if self.trace or debug: + LOG.exception(self.message) + else: + LOG.error(self.message) + + +class ValidationException(PromenadeException): + pass diff --git a/promenade/generator.py b/promenade/generator.py index e396a9a7..c68a0b91 100644 --- a/promenade/generator.py +++ b/promenade/generator.py @@ -1,486 +1,195 @@ -from . import config, logging, pki, renderer +from . import logging, pki import os +import yaml __all__ = ['Generator'] - LOG = logging.getLogger(__name__) class Generator: - @classmethod - def from_config(cls, *, config_path): - return cls(input_config=(config.load(config_path))) + def __init__(self, config, *, calico_etcd_service_ip): + self.config = config + self.calico_etcd_service_ip = calico_etcd_service_ip + self.keys = pki.PKI() + self.documents = [] - def __init__(self, *, input_config): - self.input_config = input_config + @property + def cluster_domain(self): + return self.config['KubernetesNetwork:dns.cluster_domain'] - self.validate() + def generate(self, output_dir): + # Certificate Authorities + self.gen('ca', 'kubernetes') + self.gen('ca', 'kubernetes-etcd') + self.gen('ca', 'kubernetes-etcd-peer') + self.gen('ca', 'calico-etcd') + self.gen('ca', 'calico-etcd-peer') - def validate(self): - required_kinds = ['Cluster', 'Network', 'Versions'] - for required_kind in required_kinds: - try: - self.input_config[required_kind] - except KeyError: - LOG.error('Generator requires one "%s" document to function.', - required_kind) - raise + # Certificates for Kubernetes API server + self.gen( + 'certificate', + 'apiserver', + ca='kubernetes', + cn='apiserver', + hosts=self._service_dns('kubernetes', 'default') + [ + 'localhost', '127.0.0.1', 'apiserver.kubernetes.promenade' + ] + [self.config['KubernetesNetwork:kubernetes.service_ip']]) + self.gen( + 'certificate', + 'apiserver-etcd', + ca='kubernetes-etcd', + cn='apiserver') - assert self.input_config['Cluster'].metadata['name'] \ - == self.input_config['Network'].metadata['cluster'] + # Certificates for other Kubernetes components + self.gen( + 'certificate', + 'scheduler', + ca='kubernetes', + cn='system:kube-scheduler') + self.gen( + 'certificate', + 'controller-manager', + ca='kubernetes', + cn='system:kube-controller-manager') + self.gen('keypair', 'service-account') - def generate_additional_scripts(self, output_dir): - r = renderer.Renderer(config=self.input_config, - target_dir=output_dir) - r.render_generate_files() + self.gen_kubelet_certificates() - def generate_all(self, output_dir): - self.generate_additional_scripts(output_dir) + self.gen( + 'certificate', 'proxy', ca='kubernetes', cn='system:kube-proxy') - cluster = self.input_config['Cluster'] - network = self.input_config['Network'] - versions = self.input_config['Versions'] - armada_documents = list(self.input_config.iterate(kind='ArmadaDocument')) + # Certificates for kubectl admin + self.gen( + 'certificate', + 'admin', + ca='kubernetes', + cn='admin', + groups=['system:masters']) - cluster_name = cluster.metadata['name'] - LOG.info('Generating configuration for cluster "%s"', cluster_name) - masters = self.construct_masters(cluster_name) + # Certificates for armada + self.gen( + 'certificate', + 'armada', + ca='kubernetes', + cn='armada', + groups=['system:masters']) - LOG.info('Generating common PKI for cluster "%s"', cluster_name) - keys = pki.PKI(cluster_name) - cluster_ca, cluster_ca_key = keys.generate_ca( - ca_name='cluster', - cert_target='all', - key_target='masters') - etcd_client_ca, etcd_client_ca_key = keys.generate_ca( - ca_name='etcd-client', - cert_target='masters', - key_target='masters') - etcd_peer_ca, etcd_peer_ca_key = keys.generate_ca( - ca_name='etcd-peer', - cert_target='masters', - key_target='masters') - calico_etcd_client_ca, calico_etcd_client_ca_key = keys.generate_ca( - ca_name='calico-etcd-client', - cert_target='masters', - key_target='masters') - calico_etcd_peer_ca, calico_etcd_peer_ca_key = keys.generate_ca( - ca_name='calico-etcd-peer', - cert_target='masters', - key_target='masters') + # Certificates for coredns + self.gen('certificate', 'coredns', ca='kubernetes', cn='coredns') - admin_cert, admin_cert_key = keys.generate_certificate( - name='admin', - ca_name='cluster', - groups=['system:masters'], - target='masters', - ) + # Certificates for Kubernetes's etcd servers + self.gen_etcd_certificates( + ca='kubernetes-etcd', + genesis=True, + service_name='kubernetes-etcd', + service_namespace='kube-system', + service_ip=self.config['KubernetesNetwork:etcd.service_ip'], + additional_hosts=['etcd.kubernetes.promenade']) - sa_pub, sa_priv = keys.generate_keypair( - name='service-account', - target='masters', - ) + # Certificates for Calico's etcd servers + self.gen_etcd_certificates( + ca='calico-etcd', + service_name='calico-etcd', + service_namespace='kube-system', + service_ip=self.calico_etcd_service_ip, + additional_hosts=['etcd.calico.promenade']) - config.Configuration([ - admin_cert, - admin_cert_key, - calico_etcd_client_ca, - calico_etcd_client_ca_key, - calico_etcd_peer_ca, - calico_etcd_peer_ca_key, - cluster_ca, - cluster_ca_key, - etcd_client_ca, - etcd_client_ca_key, - etcd_peer_ca, - etcd_peer_ca_key, - sa_priv, - sa_pub, - ]).write(os.path.join(output_dir, 'admin-bundle.yaml')) + # Certificates for Calico node + self.gen( + 'certificate', 'calico-node', ca='calico-etcd', cn='calico-node') - complete_configuration = [ - admin_cert, - admin_cert_key, - calico_etcd_client_ca, - calico_etcd_client_ca_key, - calico_etcd_peer_ca, - calico_etcd_peer_ca_key, - cluster_ca, - cluster_ca_key, - etcd_client_ca, - etcd_client_ca_key, - etcd_peer_ca, - etcd_peer_ca_key, - masters, - network, - sa_priv, - sa_pub, - versions, + _write(output_dir, self.documents) + + def gen(self, kind, *args, **kwargs): + method = getattr(self.keys, 'generate_' + kind) + + self.documents.extend(method(*args, **kwargs)) + + def gen_kubelet_certificates(self): + self._gen_single_kubelet( + 'genesis', node_data=self.config.get(kind='Genesis')) + for node_config in self.config.iterate(kind='KubernetesNode'): + self._gen_single_kubelet( + node_config['data']['hostname'], node_data=node_config['data']) + + def _gen_single_kubelet(self, name, node_data): + self.gen( + 'certificate', + 'kubelet-%s' % name, + ca='kubernetes', + cn='system:node:%s' % node_data['hostname'], + hosts=[node_data['hostname'], node_data['ip']], + groups=['system:nodes']) + + def gen_etcd_certificates(self, *, ca, genesis=False, **service_args): + if genesis: + self._gen_single_etcd( + name='genesis', + ca=ca, + node_data=self.config.get(kind='Genesis'), + **service_args) + + for node_config in self.config.iterate(kind='KubernetesNode'): + self._gen_single_etcd( + name=node_config['data']['hostname'], + ca=ca, + node_data=node_config['data'], + **service_args) + + self.gen( + 'certificate', + service_args['service_name'] + '-anchor', + ca=ca, + cn='anchor') + + def _gen_single_etcd(self, + *, + name, + ca, + node_data, + service_name, + service_namespace, + service_ip=None, + additional_hosts=None): + member_name = ca + '-' + name + + hosts = [ + node_data['hostname'], + node_data['ip'], + 'localhost', + '127.0.0.1', + ] + (additional_hosts or []) + + hosts.extend(self._service_dns(service_name, service_namespace)) + if service_ip is not None: + hosts.append(service_ip) + + self.gen( + 'certificate', member_name, ca=ca, cn=member_name, hosts=hosts) + + self.gen( + 'certificate', + member_name + '-peer', + ca=ca + '-peer', + cn=member_name, + hosts=hosts) + + def _service_dns(self, name, namespace): + return [ + name, + '.'.join([name, namespace]), + '.'.join([name, namespace, 'svc']), + '.'.join([name, namespace, 'svc', self.cluster_domain]), ] - for hostname, data in cluster['nodes'].items(): - if 'genesis' in data.get('roles', []): - genesis_hostname = hostname - break - for hostname, data in cluster['nodes'].items(): - LOG.debug('Generating configuration & PKI for hostname=%s', - hostname) - node = _construct_node_config(cluster_name, hostname, data) - - kubelet_cert, kubelet_cert_key = keys.generate_certificate( - alias='kubelet', - name='system:node:%s' % hostname, - ca_name='cluster', - groups=['system:nodes'], - hosts=[ - hostname, - data['ip'], - ], - target=hostname) - - proxy_cert, proxy_cert_key = keys.generate_certificate( - alias='proxy', - config_name='system:kube-proxy:%s' % hostname, - name='system:kube-proxy', - ca_name='cluster', - hosts=[ - hostname, - data['ip'], - ], - target=hostname) - - complete_configuration.extend([ - kubelet_cert, - kubelet_cert_key, - node, - proxy_cert, - proxy_cert_key, - ]) - - common_documents = [ - cluster_ca, - kubelet_cert, - kubelet_cert_key, - masters, - network, - node, - proxy_cert, - proxy_cert_key, - versions, - ] - role_specific_documents = [] - - if 'master' in data.get('roles', []): - role_specific_documents.extend([ - admin_cert, - admin_cert_key, - calico_etcd_client_ca, - calico_etcd_peer_ca, - cluster_ca_key, - etcd_client_ca, - etcd_peer_ca, - sa_priv, - sa_pub, - ]) - if 'genesis' not in data.get('roles', []): - etcd_config = _master_etcd_config( - cluster_name, genesis_hostname, hostname, masters) - calico_etcd_config = _master_calico_etcd_config( - cluster_name, genesis_hostname, hostname, masters) - complete_configuration.append(etcd_config) - complete_configuration.append(calico_etcd_config) - role_specific_documents.append(etcd_config) - role_specific_documents.append(calico_etcd_config) - master_documents = _master_config(hostname, data, - masters, network, keys) - complete_configuration.extend(master_documents) - role_specific_documents.extend(master_documents) - - if 'genesis' in data.get('roles', []): - role_specific_documents.extend(armada_documents) - role_specific_documents.extend(_genesis_config(hostname, data, - masters, network, keys)) - role_specific_documents.append(_genesis_etcd_config(cluster_name, hostname)) - role_specific_documents.append(_genesis_calico_etcd_config(cluster_name, hostname)) - node.data['spec']['is_genesis'] = True - - c = config.Configuration(common_documents + role_specific_documents) - c.write(os.path.join(output_dir, hostname + '.yaml')) - - config.Configuration(complete_configuration).write( - os.path.join(output_dir, 'complete-bundle.yaml')) - - def construct_masters(self, cluster_name): - masters = [] - for hostname, data in self.input_config['Cluster']['nodes'].items(): - if 'master' in data.get('roles', []) or 'genesis' in data.get('roles', []): - masters.append({'hostname': hostname, 'ip': data['ip']}) - - return config.Document({ - 'apiVersion': 'promenade/v1', - 'kind': 'Masters', - 'metadata': { - 'cluster': cluster_name, - 'name': cluster_name, - 'target': 'all', - }, - 'spec': { - 'nodes': masters, - }, - }) - - -def _master_etcd_config(cluster_name, genesis_hostname, hostname, masters): - initial_cluster = ['%s=https://%s:2380' % (m['hostname'], - m['hostname']) - for m in masters['nodes']] - initial_cluster.extend([ - 'auxiliary-etcd-0=https://%s:12380' % genesis_hostname, - 'auxiliary-etcd-1=https://%s:22380' % genesis_hostname, - ]) - return _etcd_config(cluster_name, alias='kube-etcd', - name='master-kube-etcd:%s' % hostname, - target=hostname, - initial_cluster=initial_cluster, - initial_cluster_state='existing') - - -def _master_calico_etcd_config(cluster_name, genesis_hostname, hostname, masters): - initial_cluster = ['%s=https://%s:6667' % (m['hostname'], - m['hostname']) - for m in masters['nodes']] - initial_cluster.extend([ - 'auxiliary-calico-etcd-0=https://%s:16667' % genesis_hostname, - 'auxiliary-calico-etcd-1=https://%s:26667' % genesis_hostname, - ]) - return _etcd_config(cluster_name, alias='calico-etcd', - name='master-calico-etcd:%s' % hostname, - target=hostname, - initial_cluster=initial_cluster, - initial_cluster_state='existing') - - -def _genesis_etcd_config(cluster_name, hostname): - initial_cluster = [ - '%s=https://%s:2380' % (hostname, hostname), - 'auxiliary-etcd-0=https://%s:12380' % hostname, - 'auxiliary-etcd-1=https://%s:22380' % hostname, - ] - return _etcd_config(cluster_name, alias='kube-etcd', - name='master-kube-etcd:%s' % hostname, - target=hostname, - initial_cluster=initial_cluster, - initial_cluster_state='new') - - -def _genesis_calico_etcd_config(cluster_name, hostname): - initial_cluster = [ - '%s=https://%s:6667' % (hostname, hostname), - 'auxiliary-calico-etcd-0=https://%s:16667' % hostname, - 'auxiliary-calico-etcd-1=https://%s:26667' % hostname, - ] - return _etcd_config(cluster_name, alias='calico-etcd', - name='master-calico-etcd:%s' % hostname, - target=hostname, - initial_cluster=initial_cluster, - initial_cluster_state='new') - - -def _etcd_config(cluster_name, *, alias, name, target, - initial_cluster, initial_cluster_state): - return config.Document({ - 'apiVersion': 'promenade/v1', - 'kind': 'Etcd', - 'metadata': { - 'cluster': cluster_name, - 'alias': alias, - 'name': name, - 'target': target, - }, - 'spec': { - 'initial_cluster': initial_cluster, - 'initial_cluster_state': initial_cluster_state, - }, - }) - - -def _master_config(hostname, host_data, masters, network, keys): - kube_domains = [ - 'kubernetes', - 'kubernetes.default', - 'kubernetes.default.svc', - 'kubernetes.default.svc.cluster.local', - '127.0.0.1', - ] - calico_domains = [ - 'calico-etcd', - 'calico-etcd.kube-system', - 'calico-etcd.kube-system.svc', - 'calico-etcd.kube-system.svc.cluster.local', - network['calico_etcd_service_ip'], - ] - - docs = [] - - docs.extend(keys.generate_certificate( - alias='etcd-client', - name='etcd:client:%s' % hostname, - ca_name='etcd-client', - hosts=kube_domains + calico_domains + [hostname, host_data['ip']], - target=hostname, - )) - docs.extend(keys.generate_certificate( - alias='etcd-apiserver-client', - name='etcd:client:apiserver:%s' % hostname, - ca_name='etcd-client', - hosts=[hostname, host_data['ip']], - target=hostname, - )) - - docs.extend(keys.generate_certificate( - alias='etcd-peer', - name='etcd:peer:%s' % hostname, - ca_name='etcd-peer', - hosts=kube_domains + [hostname, host_data['ip']], - target=hostname, - )) - - docs.extend(keys.generate_certificate( - alias='calico-etcd-client', - name='calico-etcd:client:%s' % hostname, - ca_name='calico-etcd-client', - hosts=kube_domains + calico_domains + [hostname, host_data['ip']], - target=hostname, - )) - - docs.extend(keys.generate_certificate( - alias='calico-etcd-peer', - name='calico-etcd:peer:%s' % hostname, - ca_name='calico-etcd-peer', - hosts=kube_domains + [hostname, host_data['ip']], - target=hostname, - )) - - docs.extend(keys.generate_certificate( - alias='apiserver', - name='apiserver:%s' % hostname, - ca_name='cluster', - hosts=kube_domains + [ - network['kube_service_ip'], - hostname, - host_data['ip'], - ], - target=hostname, - )) - - docs.extend(keys.generate_certificate( - alias='controller-manager', - config_name='system:kube-controller-manager:%s' % hostname, - name='system:kube-controller-manager', - ca_name='cluster', - hosts=[ - hostname, - host_data['ip'], - ], - target=hostname, - )) - - docs.extend(keys.generate_certificate( - alias='scheduler', - config_name='system:kube-scheduler:%s' % hostname, - name='system:kube-scheduler', - ca_name='cluster', - hosts=[ - hostname, - host_data['ip'], - ], - target=hostname, - )) - - return docs - - -def _genesis_config(hostname, host_data, masters, network, keys): - docs = [] - - for i in range(2): - docs.extend(keys.generate_certificate( - name='auxiliary-etcd-%d-client' % i, - ca_name='etcd-client', - hosts=[hostname, host_data['ip']], - target=hostname, - )) - - docs.extend(keys.generate_certificate( - name='auxiliary-etcd-%d-peer' % i, - ca_name='etcd-peer', - hosts=[hostname, host_data['ip']], - target=hostname, - )) - - docs.extend(keys.generate_certificate( - name='auxiliary-calico-etcd-%d-client' % i, - ca_name='calico-etcd-client', - hosts=[hostname, host_data['ip']], - target=hostname, - )) - - docs.extend(keys.generate_certificate( - name='auxiliary-calico-etcd-%d-peer' % i, - ca_name='calico-etcd-peer', - hosts=[hostname, host_data['ip']], - target=hostname, - )) - - docs.extend(keys.generate_certificate( - alias='calico-etcd-node-client', - name='calico-etcd:node', - ca_name='calico-etcd-client', - target=hostname, - )) - - return docs - - -def _construct_node_config(cluster_name, hostname, data): - spec = { - 'hostname': hostname, - 'ip': data['ip'], - 'labels': _labels(data.get('roles', []), data.get('additional_labels', [])), - 'templates': _templates(data.get('roles', [])), - } - - return config.Document({ - 'apiVersion': 'promenade/v1', - 'kind': 'Node', - 'metadata': { - 'cluster': cluster_name, - 'name': hostname, - 'target': hostname, - }, - 'spec': spec, - }) - - -ROLE_LABELS = { - 'genesis': [ - 'promenade=genesis', - ], - 'master': [ - 'node-role.kubernetes.io/master=', - ], -} - - -def _labels(roles, additional_labels): - result = set() - for role in roles: - result.update(ROLE_LABELS.get(role, [])) - result.update(additional_labels) - return sorted(result) - - -def _templates(roles): - return ['common'] + roles +def _write(output_dir, docs): + with open(os.path.join(output_dir, 'certificates.yaml'), 'w') as f: + # Don't use safe_dump_all so we can block format certificate data. + yaml.dump_all( + docs, + stream=f, + default_flow_style=False, + explicit_start=True, + indent=2) diff --git a/promenade/logging.py b/promenade/logging.py index 5976afbf..c8d73bc6 100644 --- a/promenade/logging.py +++ b/promenade/logging.py @@ -3,8 +3,7 @@ from logging import getLogger __all__ = ['getLogger', 'setup'] - -LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(funcName)s [%(lineno)3d] %(message)s' +LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(funcName)s [%(lineno)3d] %(message)s' # noqa def setup(*, verbose): diff --git a/promenade/operator.py b/promenade/operator.py deleted file mode 100644 index 10dad544..00000000 --- a/promenade/operator.py +++ /dev/null @@ -1,48 +0,0 @@ -from . import config, logging, renderer -import os -import subprocess - -__all__ = ['Operator'] - - -LOG = logging.getLogger(__name__) - - -class Operator: - @classmethod - def from_config(cls, *, config_path, hostname, target_dir): - return cls(hostname=hostname, target_dir=target_dir, - config_=config.load(config_path)) - - def __init__(self, *, config_, hostname, target_dir): - self.config = config_ - self.hostname = hostname - self.target_dir = target_dir - - def up(self, *, asset_dir): - self.rsync_from(asset_dir) - self.render() - - self.bootstrap() - - def rsync_from(self, src): - if src: - LOG.debug('Syncing assets from "%s" to "%s".', src, self.target_dir) - subprocess.run(['/usr/bin/rsync', '-r', - os.path.join(src, ''), self.target_dir], - check=True) - else: - LOG.debug('No source directory given for rsync.') - - - def render(self): - r = renderer.Renderer(config=self.config, - target_dir=self.target_dir) - r.render() - - def bootstrap(self): - LOG.debug('Running genesis script with chroot "%s"', self.target_dir) - subprocess.run([os.path.join(self.target_dir, 'usr/sbin/chroot'), - self.target_dir, - '/bin/bash', '/usr/local/bin/bootstrap'], - check=True) diff --git a/promenade/pki.py b/promenade/pki.py index 31266508..4086dca0 100644 --- a/promenade/pki.py +++ b/promenade/pki.py @@ -1,4 +1,4 @@ -from . import config, logging +from . import logging import json import os import subprocess @@ -7,18 +7,13 @@ import yaml __all__ = ['PKI'] - LOG = logging.getLogger(__name__) class PKI: - def __init__(self, cluster_name, *, ca_config=None): + def __init__(self): self.certificate_authorities = {} - self.cluster_name = cluster_name - self._ca_config_string = None - if ca_config: - self._ca_config_string = json.dumps(ca_config) @property def ca_config(self): @@ -26,86 +21,70 @@ class PKI: self._ca_config_string = json.dumps({ 'signing': { 'default': { - 'expiry': '8760h', - 'usages': ['signing', 'key encipherment', 'server auth', 'client auth'], + 'expiry': + '8760h', + 'usages': [ + 'signing', 'key encipherment', 'server auth', + 'client auth' + ], }, }, }) return self._ca_config_string - def generate_ca(self, *, ca_name, cert_target, key_target): - result = self._cfssl(['gencert', '-initca', 'csr.json'], - files={ - 'csr.json': self.csr( - name=ca_name, - groups=['Kubernetes']), - }) + def generate_ca(self, ca_name): + result = self._cfssl( + ['gencert', '-initca', 'csr.json'], + files={ + 'csr.json': self.csr(name=ca_name, groups=['Kubernetes']), + }) LOG.debug('ca_cert=%r', result['cert']) self.certificate_authorities[ca_name] = result - return (self._wrap('CertificateAuthority', result['cert'], - name=ca_name, - target=cert_target), - self._wrap('CertificateAuthorityKey', result['key'], - name=ca_name, - target=key_target)) + return (self._wrap_ca(ca_name, result['cert']), self._wrap_ca_key( + ca_name, result['key'])) - def generate_keypair(self, *, alias=None, name, target): + def generate_keypair(self, name): priv_result = self._openssl(['genrsa', '-out', 'priv.pem']) - pub_result = self._openssl(['rsa', '-in', 'priv.pem', '-pubout', '-out', 'pub.pem'], - files={ - 'priv.pem': priv_result['priv.pem'], - }) + pub_result = self._openssl( + ['rsa', '-in', 'priv.pem', '-pubout', '-out', 'pub.pem'], + files={ + 'priv.pem': priv_result['priv.pem'], + }) - if not alias: - alias = name + return (self._wrap_pub_key(name, pub_result['pub.pem']), + self._wrap_priv_key(name, priv_result['priv.pem'])) - return (self._wrap('PublicKey', pub_result['pub.pem'], - alias=alias, - name=name, - target=target), - self._wrap('PrivateKey', priv_result['priv.pem'], - alias=alias, - name=name, - target=target)) - - - def generate_certificate(self, *, alias=None, config_name=None, - ca_name, groups=[], hosts=[], name, target): + def generate_certificate(self, name, *, ca, cn, groups=[], hosts=[]): result = self._cfssl( - ['gencert', - '-ca', 'ca.pem', - '-ca-key', 'ca-key.pem', - '-config', 'ca-config.json', - 'csr.json'], - files={ - 'ca-config.json': self.ca_config, - 'ca.pem': self.certificate_authorities[ca_name]['cert'], - 'ca-key.pem': self.certificate_authorities[ca_name]['key'], - 'csr.json': self.csr(name=name, groups=groups, hosts=hosts), - }) + [ + 'gencert', '-ca', 'ca.pem', '-ca-key', 'ca-key.pem', '-config', + 'ca-config.json', 'csr.json' + ], + files={ + 'ca-config.json': self.ca_config, + 'ca.pem': self.certificate_authorities[ca]['cert'], + 'ca-key.pem': self.certificate_authorities[ca]['key'], + 'csr.json': self.csr(name=cn, groups=groups, hosts=hosts), + }) - if not alias: - alias = name + return (self._wrap_cert(name, result['cert']), self._wrap_cert_key( + name, result['key'])) - if not config_name: - config_name = name - - return (self._wrap('Certificate', result['cert'], - alias=alias, - name=config_name, - target=target), - self._wrap('CertificateKey', result['key'], - alias=alias, - name=config_name, - target=target)) - - def csr(self, *, name, groups=[], hosts=[], key={'algo': 'rsa', 'size': 2048}): + def csr(self, + *, + name, + groups=[], + hosts=[], + key={'algo': 'rsa', + 'size': 2048}): return json.dumps({ 'CN': name, 'key': key, 'hosts': hosts, - 'names': [{'O': g} for g in groups], + 'names': [{ + 'O': g + } for g in groups], }) def _cfssl(self, command, *, files=None): @@ -116,8 +95,8 @@ class PKI: with open(os.path.join(tmp, filename), 'w') as f: f.write(data) - return json.loads(subprocess.check_output( - ['cfssl'] + command, cwd=tmp)) + return json.loads( + subprocess.check_output(['cfssl'] + command, cwd=tmp)) def _openssl(self, command, *, files=None): if not files: @@ -138,25 +117,45 @@ class PKI: return result - def _wrap(self, kind, data, **metadata): - return config.Document({ - 'apiVersion': 'promenade/v1', - 'kind': kind, + def _wrap_ca(self, name, data): + return self._wrap(kind='CertificateAuthority', name=name, data=data) + + def _wrap_ca_key(self, name, data): + return self._wrap(kind='CertificateAuthorityKey', name=name, data=data) + + def _wrap_cert(self, name, data): + return self._wrap(kind='Certificate', name=name, data=data) + + def _wrap_cert_key(self, name, data): + return self._wrap(kind='CertificateKey', name=name, data=data) + + def _wrap_priv_key(self, name, data): + return self._wrap(kind='PrivateKey', name=name, data=data) + + def _wrap_pub_key(self, name, data): + return self._wrap(kind='PublicKey', name=name, data=data) + + def _wrap(self, *, data, kind, name): + return { + 'schema': 'deckhand/%s/v1' % kind, 'metadata': { - 'cluster': self.cluster_name, - **metadata, + 'schema': 'metadata/Document/v1', + 'name': name, + 'layerinDefinition': { + 'abstract': False, + 'layer': 'site', + }, }, - 'spec': { - 'data': block_literal(data), - }, - }) + 'data': block_literal(data), + } -class block_literal(str): pass +class block_literal(str): + pass def block_literal_representer(dumper, data): - return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|') + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') yaml.add_representer(block_literal, block_literal_representer) diff --git a/promenade/renderer.py b/promenade/renderer.py index 785dd019..a4e1bea9 100644 --- a/promenade/renderer.py +++ b/promenade/renderer.py @@ -1,71 +1,140 @@ -from . import logging +from . import logging, tar_bundler import base64 +import datetime import io import jinja2 import os import pkg_resources import yaml -__all__ = ['Renderer'] - +__all__ = [ + 'build_tarball_from_roles', 'insert_charts_into_bundler', + 'render_role_into_bundler' +] LOG = logging.getLogger(__name__) -class Renderer: - def __init__(self, *, config, target_dir): - self.config = config - self.target_dir = target_dir +def build_tarball_from_roles(config, *, roles, file_specs): + bundler = tar_bundler.TarBundler() + for role in roles: + render_role_into_bundler(bundler=bundler, config=config, role=role) - def render_generate_files(self): - self.render_template_dir('generate') + for file_spec in file_specs: + bundler.add(**file_spec) - def render(self): - for template_dir in self.config['Node']['templates']: - self.render_template_dir(template_dir) + if 'genesis' in roles: + insert_charts_into_bundler(bundler) - def render_template_dir(self, template_dir): - source_root = pkg_resources.resource_filename( - 'promenade', os.path.join('templates', template_dir)) - LOG.debug('Searching for templates in: "%s"', source_root) - for root, _dirnames, filenames in os.walk(source_root, - followlinks=True): - for source_filename in filenames: - source_path = os.path.join(root, source_filename) - self.render_template_file(path=source_path, - root=source_root) - - def render_template_file(self, *, path, root): - base_path = os.path.relpath(path, root) - target_path = os.path.join(self.target_dir, base_path) - - _ensure_path(target_path) - - LOG.debug('Templating "%s" into "%s"', path, target_path) - - env = jinja2.Environment( - loader=jinja2.PackageLoader('promenade', 'templates/include'), - undefined=jinja2.StrictUndefined) - env.filters['b64enc'] = _base64_encode - env.filters['yaml_safe_dump_all'] = _yaml_safe_dump_all - - with open(path) as f: - template = env.from_string(f.read()) - rendered_data = template.render(config=self.config) - - with open(target_path, 'w') as f: - f.write(rendered_data) - - LOG.info('Installed "%s"', os.path.join('/', base_path)) + return bundler.as_blob() -def _ensure_path(path): - base = os.path.dirname(path) - os.makedirs(base, mode=0o775, exist_ok=True) +def insert_charts_into_bundler(bundler): + for root, _dirnames, filenames in os.walk( + '/promenade/charts', followlinks=True): + for source_filename in filenames: + source_path = os.path.join(root, source_filename) + destination_path = os.path.join('etc/genesis/armada/assets/charts', + os.path.relpath( + source_path, + '/promenade/charts')) + stat = os.stat(source_path) + LOG.debug('Copying asset file %s (mode=%o)', source_path, + stat.st_mode) + with open(source_path) as f: + bundler.add( + path=destination_path, data=f.read(), mode=stat.st_mode) + + +def render_role_into_bundler(*, bundler, config, role): + role_root = pkg_resources.resource_filename('promenade', + os.path.join( + 'templates', 'roles', + role)) + for root, _dirnames, filenames in os.walk(role_root, followlinks=True): + destination_base = os.path.relpath(root, role_root) + for source_filename in filenames: + source_path = os.path.join(root, source_filename) + stat = os.stat(source_path) + LOG.debug('Rendering file %s (mode=%o)', source_path, stat.st_mode) + destination_path = os.path.join(destination_base, source_filename) + render_template_into_bundler( + bundler=bundler, + config=config, + destination_path=destination_path, + source_path=source_path, + mode=stat.st_mode) + + +def render_template_into_bundler(*, bundler, config, destination_path, + source_path, mode): + env = _build_env() + + with open(source_path) as f: + template = env.from_string(f.read()) + now = int(datetime.datetime.utcnow().timestamp()) + data = template.render(config=config, now=now) + bundler.add(path=destination_path, data=data, mode=mode) + + +def render_template(config, *, template, context=None): + if context is None: + context = {} + + template_contents = pkg_resources.resource_string('promenade', + os.path.join( + 'templates', + template)) + + env = _build_env() + + template_obj = env.from_string(template_contents.decode('utf-8')) + return template_obj.render(config=config, **context) + + +def _build_env(): + env = jinja2.Environment( + loader=jinja2.PackageLoader('promenade', 'templates/include'), + undefined=jinja2.StrictUndefined) + env.filters['b64enc'] = _base64_encode + env.filters['fill_no_proxy'] = _fill_no_proxy + env.filters['yaml_safe_dump_all'] = _yaml_safe_dump_all + return env def _base64_encode(s): - return base64.b64encode(s.encode()).decode() + try: + return base64.b64encode(s).decode() + except TypeError: + return base64.b64encode(s.encode()).decode() + + +def _fill_no_proxy(network_config): + proxy = network_config.get('proxy', {}).get('url') + if proxy: + additional = network_config.get('proxy', {}).get( + 'additional_no_proxy', []) + if additional: + return ','.join(additional) + ',' + _default_no_proxy( + network_config) + else: + return _default_no_proxy(network_config) + else: + return '' + + +def _default_no_proxy(network_config): + # XXX We can add better default data. + include = [ + '127.0.0.1', + 'localhost', + 'kubernetes', + 'kubernetes.default', + 'kubernetes.default.svc', + 'kubernetes.default.svc.%s' % network_config.get('dns', {}).get( + 'cluster_domain', 'cluster.local'), + ] + return ','.join(include) def _yaml_safe_dump_all(documents): diff --git a/promenade/schemas/ArmadaPlaceholders.yaml b/promenade/schemas/ArmadaPlaceholders.yaml new file mode 100644 index 00000000..a5c6d82a --- /dev/null +++ b/promenade/schemas/ArmadaPlaceholders.yaml @@ -0,0 +1,28 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: armada/Manifest/v1 + labels: + application: armada +data: + $schema: http://json-schema.org/schema# +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: armada/ChartGroup/v1 + labels: + application: armada +data: + $schema: http://json-schema.org/schema# +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: armada/Chart/v1 + labels: + application: armada +data: + $schema: http://json-schema.org/schema# +... diff --git a/promenade/schemas/DeckhandPlaceholders.yaml b/promenade/schemas/DeckhandPlaceholders.yaml new file mode 100644 index 00000000..cdc4008a --- /dev/null +++ b/promenade/schemas/DeckhandPlaceholders.yaml @@ -0,0 +1,61 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: deckhand/Certificate/v1 + labels: + application: deckhand +data: + $schema: http://json-schema.org/schema# + type: string +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: deckhand/CertificateKey/v1 + labels: + application: deckhand +data: + $schema: http://json-schema.org/schema# + type: string +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: deckhand/CertificateAuthority/v1 + labels: + application: deckhand +data: + $schema: http://json-schema.org/schema# + type: string +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: deckhand/CertificateAuthorityKey/v1 + labels: + application: deckhand +data: + $schema: http://json-schema.org/schema# + type: string +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: deckhand/PublicKey/v1 + labels: + application: deckhand +data: + $schema: http://json-schema.org/schema# + type: string +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: deckhand/PrivateKey/v1 + labels: + application: deckhand +data: + $schema: http://json-schema.org/schema# + type: string +... diff --git a/promenade/schemas/Docker.yaml b/promenade/schemas/Docker.yaml new file mode 100644 index 00000000..9f034fe2 --- /dev/null +++ b/promenade/schemas/Docker.yaml @@ -0,0 +1,17 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: promenade/Docker/v1 + labels: + application: promenade +data: + $schema: http://json-schema.org/schema# + type: object + properties: + config: + type: object + required: + - config + additionalProperties: false +... diff --git a/promenade/schemas/Genesis.yaml b/promenade/schemas/Genesis.yaml new file mode 100644 index 00000000..0e70db8b --- /dev/null +++ b/promenade/schemas/Genesis.yaml @@ -0,0 +1,127 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: promenade/Genesis/v1 + labels: + application: promenade +data: + $schema: http://json-schema.org/schema# + definitions: + abs_path: + type: string + pattern: '^/.+$' + hostname: + type: string + pattern: '^[a-z][a-z0-9-]+$' + file: + properties: + path: + $ref: '#/definitions/abs_path' + content: + type: string + mode: + type: integer + minimum: 0 + tar_url: + $ref: '#/definitions/url' + tar_path: + $ref: '#/definitions/rel_path' + + requried: + - mode + - path + oneOf: + - type: object + required: + - content + - type: object + allOf: + - type: object + required: + - tar_url + - tar_path + additionalProperties: false + image: + type: string + # XXX add regex + ip_address: + type: string + pattern: '^(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))$' + kubernetes_label: + type: string + # XXX add regex + rel_path: + type: string + # XXX add regex + + type: object + properties: + files: + type: array + items: + $ref: '#/definitions/file' + + hostname: + $ref: '#/definitions/hostname' + + ip: + $ref: '#/definitions/ip_address' + + labels: + properties: + static: + type: array + items: + $ref: '#/definitions/kubernetes_label' + dynamic: + type: array + items: + $ref: '#/definitions/kubernetes_label' + required: + - static + additionalProperties: false + + images: + type: object + properties: + armada: + $ref: '#/definitions/image' + helm: + type: object + properties: + tiller: + $ref: '#/definitions/image' + required: + - tiller + additionalProperties: false + kubernetes: + type: object + properties: + apiserver: + $ref: '#/definitions/image' + controller-manager: + $ref: '#/definitions/image' + etcd: + $ref: '#/definitions/image' + scheduler: + $ref: '#/definitions/image' + required: + - apiserver + - controller-manager + - etcd + - scheduler + additionalProperties: false + required: + - armada + - helm + - kubernetes + additionalProperties: false + + required: + - hostname + - ip + - images + - labels + additionalProperties: false +... diff --git a/promenade/schemas/HostSystem.yaml b/promenade/schemas/HostSystem.yaml new file mode 100644 index 00000000..56629b6f --- /dev/null +++ b/promenade/schemas/HostSystem.yaml @@ -0,0 +1,134 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: promenade/HostSystem/v1 + labels: + application: promenade +data: + $schema: http://json-schema.org/schema# + definitions: + abs_path: + type: string + pattern: '^/.+$' + apt_source_line: + type: string + # XXX add regex + file: + properties: + path: + $ref: '#/definitions/abs_path' + content: + type: string + mode: + type: integer + minimum: 0 + tar_url: + $ref: '#/definitions/url' + tar_path: + $ref: '#/definitions/rel_path' + + requried: + - mode + - path + oneOf: + - type: object + required: + - content + - type: object + allOf: + - type: object + required: + - tar_url + - tar_path + additionalProperties: false + + image: + type: string + # XXX add regex + package: + type: string + # XXX add regex + public_key: + type: string + # XXX add regex + rel_path: + type: string + # XXX add regex + url: + type: string + # XXX add regex + + type: object + + properties: + files: + type: array + items: + type: object + items: + $ref: '#/definitions/file' + images: + type: object + properties: + coredns: + $ref: '#/definitions/image' + helm: + type: object + properties: + helm: + $ref: '#/definitions/image' + required: + - helm + additionalProperties: false + kubernetes: + type: object + properties: + kubectl: + $ref: '#/definitions/image' + required: + - kubectl + additionalProperties: false + required: + - coredns + - helm + - kubernetes + additionalProperties: false + + packages: + type: object + properties: + additional: + type: array + items: + $ref: '#/definitions/package' + keys: + type: array + items: + $ref: '#/definitions/public_key' + + required: + type: object + properties: + docker: + $ref: '#/definitions/package' + socat: + $ref: '#/definitions/package' + required: + - docker + - socat + additionalProperties: false + + repositories: + type: array + items: + $ref: '#/definitions/apt_source_line' + + required: + - required + additionalProperties: false + + required: + - images + - packages + additionalProperties: false diff --git a/promenade/schemas/KubernetesNetwork.yaml b/promenade/schemas/KubernetesNetwork.yaml new file mode 100644 index 00000000..d66fbc9b --- /dev/null +++ b/promenade/schemas/KubernetesNetwork.yaml @@ -0,0 +1,104 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: promenade/KubernetesNetwork/v1 + labels: + application: promenade +data: + $schema: http://json-schema.org/schema# + definitions: + cidr: + type: string + pattern: '^(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\/([0-9]|[1-2][0-9]|3[0-2])$' + domain_name: + type: string + pattern: '^([a-z][a-z0-9-]+\.)+[a-z]+\.?$' + hostname: + type: string + pattern: '^([a-z][a-z0-9-]+)(\.+[a-z]+\.)?$' + hostname_or_ip_address: + type: string + pattern: '^(([a-z][a-z0-9-]+)(\.+[a-z]+\.)?|(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5])))$' + ip_address: + type: string + pattern: '^(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))$' + url: + type: string + # XXX add regex + + type: object + properties: + dns: + type: object + properties: + bootstrap_validation_checks: + type: array + items: + $ref: '#/definitions/domain_name' + cluster_domain: + $ref: '#/definitions/domain_name' + service_ip: + $ref: '#/definitions/ip_address' + upstream_servers: + type: array + items: + $ref: '#/definitions/ip_address' + required: + - cluster_domain + - service_ip + additionalProperties: false + + etcd: + type: object + properties: + service_ip: + $ref: '#/definitions/ip_address' + required: + - service_ip + additionalProperties: false + + kubernetes: + type: object + properties: + pod_cidr: + $ref: '#/definitions/cidr' + service_ip: + $ref: '#/definitions/ip_address' + service_cidr: + $ref: '#/definitions/cidr' + required: + - pod_cidr + - service_cidr + - service_ip + additionalProperties: false + hosts_entries: + type: array + items: + type: object + properties: + ip: + $ref: '#/definitions/ip_address' + names: + type: array + items: + $ref: '#/definitions/hostname' + + proxy: + type: object + properties: + additional_no_proxy: + type: array + items: + $ref: '#/definitions/hostname_or_ip_address' + url: + $ref: '#/definitions/url' + required: + - url + additionalFields: false + + required: + - dns + - kubernetes + additionalProperties: false +... diff --git a/promenade/schemas/KubernetesNode.yaml b/promenade/schemas/KubernetesNode.yaml new file mode 100644 index 00000000..34a29571 --- /dev/null +++ b/promenade/schemas/KubernetesNode.yaml @@ -0,0 +1,48 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: promenade/KubernetesNode/v1 + labels: + application: promenade +data: + $schema: http://json-schema.org/schema# + definitions: + hostname: + type: string + pattern: '^[a-z][a-z0-9-]+$' + ip_address: + type: string + pattern: '^(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))\.(\d|[1-9]\d|1\d\d|2([0-4]\d|5[0-5]))$' + kubernetes_label: + type: string + # XXX add regex + + type: object + properties: + hostname: + $ref: '#/definitions/hostname' + + ip: + $ref: '#/definitions/ip_address' + + join_ip: + $ref: '#/definitions/ip_address' + + labels: + properties: + static: + type: array + items: + $ref: '#/definitions/kubernetes_label' + dynamic: + type: array + items: + $ref: '#/definitions/kubernetes_label' + additionalProperties: false + + required: + - ip + - join_ip + additionalProperties: false +... diff --git a/promenade/tar_bundler.py b/promenade/tar_bundler.py new file mode 100644 index 00000000..d355c03f --- /dev/null +++ b/promenade/tar_bundler.py @@ -0,0 +1,38 @@ +from . import logging +import hashlib +import io +import tarfile + +__all__ = ['TarBundler'] + +LOG = logging.getLogger(__name__) + + +class TarBundler: + def __init__(self): + self._tar_blob = io.BytesIO() + self._tf = tarfile.open(fileobj=self._tar_blob, mode='w|gz') + + def add(self, *, path, data, mode): + if path.startswith('/'): + path = path[1:] + + tar_info = tarfile.TarInfo(name=path) + if isinstance(data, str): + data_bytes = data.encode('utf-8') + else: + data_bytes = data + tar_info.size = len(data_bytes) + tar_info.mode = mode + + if tar_info.size > 0: + LOG.debug('Adding file path=%s size=%s md5=%s', path, + tar_info.size, hashlib.md5(data_bytes).hexdigest()) + else: + LOG.warning('Zero length file added to path=%s', path) + + self._tf.addfile(tar_info, io.BytesIO(data_bytes)) + + def as_blob(self): + self._tf.close() + return self._tar_blob.getvalue() diff --git a/promenade/templates/common/etc/apt/apt.conf.d/20-proxy.conf b/promenade/templates/common/etc/apt/apt.conf.d/20-proxy.conf deleted file mode 100644 index 3003912e..00000000 --- a/promenade/templates/common/etc/apt/apt.conf.d/20-proxy.conf +++ /dev/null @@ -1,7 +0,0 @@ -#Promenade Generated -{% if config['Network']['http_proxy'] is defined %} -Acquire::http::Proxy "{{ config['Network']['http_proxy'] }}"; -{% endif %} -{% if config['Network']['https_proxy'] is defined %} -Acquire::https::Proxy "{{ config['Network']['https_proxy'] }}"; -{% endif %} diff --git a/promenade/templates/common/etc/dnsmasq.d/kubernetes-masters b/promenade/templates/common/etc/dnsmasq.d/kubernetes-masters deleted file mode 100644 index ad9fa7c3..00000000 --- a/promenade/templates/common/etc/dnsmasq.d/kubernetes-masters +++ /dev/null @@ -1,4 +0,0 @@ -{% for master in config['Masters']['nodes'] %} -host-record=kubernetes,{{ master['ip'] }} -host-record={{ master['hostname'] }},{{ master['ip'] }} -{% endfor %} diff --git a/promenade/templates/common/etc/dnsmasq.d/upstream-dns b/promenade/templates/common/etc/dnsmasq.d/upstream-dns deleted file mode 100644 index a46d1219..00000000 --- a/promenade/templates/common/etc/dnsmasq.d/upstream-dns +++ /dev/null @@ -1 +0,0 @@ -server=/{{ config['Network']['cluster_domain'] }}/{{ config['Network']['cluster_dns'] }} diff --git a/promenade/templates/common/etc/kubernetes/kubelet/manifests/kube-proxy.yaml b/promenade/templates/common/etc/kubernetes/kubelet/manifests/kube-proxy.yaml deleted file mode 100644 index 71b7fc79..00000000 --- a/promenade/templates/common/etc/kubernetes/kubelet/manifests/kube-proxy.yaml +++ /dev/null @@ -1,39 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: kube-proxy - namespace: kube-system - labels: - tier: node - component: kube-proxy - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' -spec: - containers: - - name: kube-proxy - image: {{ config['Versions']['images']['kubernetes']['proxy'] }} - command: - - /hyperkube - - proxy - - --cluster-cidr={{ config['Network']['pod_ip_cidr'] }} - - --hostname-override=$(NODE_NAME) - - --kubeconfig=/etc/kubernetes/proxy/kubeconfig.yaml - - --proxy-mode=iptables - - --v=5 - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: true - volumeMounts: - - name: config - mountPath: /etc/kubernetes/proxy - readOnly: true - hostNetwork: true - volumes: - - name: config - hostPath: - path: /etc/kubernetes/proxy diff --git a/promenade/templates/common/etc/kubernetes/kubelet/pki/cluster-ca.pem b/promenade/templates/common/etc/kubernetes/kubelet/pki/cluster-ca.pem deleted file mode 100644 index 51adb572..00000000 --- a/promenade/templates/common/etc/kubernetes/kubelet/pki/cluster-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='cluster')['data'] }} diff --git a/promenade/templates/common/etc/kubernetes/kubelet/pki/kubelet-key.pem b/promenade/templates/common/etc/kubernetes/kubelet/pki/kubelet-key.pem deleted file mode 100644 index 4d4655bf..00000000 --- a/promenade/templates/common/etc/kubernetes/kubelet/pki/kubelet-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', alias='kubelet')['data'] }} diff --git a/promenade/templates/common/etc/kubernetes/kubelet/pki/kubelet.pem b/promenade/templates/common/etc/kubernetes/kubelet/pki/kubelet.pem deleted file mode 100644 index 2926699d..00000000 --- a/promenade/templates/common/etc/kubernetes/kubelet/pki/kubelet.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', alias='kubelet')['data'] }} diff --git a/promenade/templates/common/etc/kubernetes/proxy/kubeconfig.yaml b/promenade/templates/common/etc/kubernetes/proxy/kubeconfig.yaml deleted file mode 100644 index 0c40dd43..00000000 --- a/promenade/templates/common/etc/kubernetes/proxy/kubeconfig.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: v1 -clusters: -- cluster: - server: https://kubernetes:6443 - certificate-authority: /etc/kubernetes/proxy/pki/cluster-ca.pem - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: proxy - name: proxy@kubernetes -current-context: proxy@kubernetes -kind: Config -preferences: {} -users: -- name: proxy - user: - client-certificate: /etc/kubernetes/proxy/pki/proxy.pem - client-key: /etc/kubernetes/proxy/pki/proxy-key.pem diff --git a/promenade/templates/common/etc/kubernetes/proxy/pki/cluster-ca.pem b/promenade/templates/common/etc/kubernetes/proxy/pki/cluster-ca.pem deleted file mode 100644 index 51adb572..00000000 --- a/promenade/templates/common/etc/kubernetes/proxy/pki/cluster-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='cluster')['data'] }} diff --git a/promenade/templates/common/etc/kubernetes/proxy/pki/proxy-key.pem b/promenade/templates/common/etc/kubernetes/proxy/pki/proxy-key.pem deleted file mode 100644 index 22ad08f9..00000000 --- a/promenade/templates/common/etc/kubernetes/proxy/pki/proxy-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', alias='proxy')['data'] }} diff --git a/promenade/templates/common/etc/kubernetes/proxy/pki/proxy.pem b/promenade/templates/common/etc/kubernetes/proxy/pki/proxy.pem deleted file mode 100644 index 0587e040..00000000 --- a/promenade/templates/common/etc/kubernetes/proxy/pki/proxy.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', alias='proxy')['data'] }} diff --git a/promenade/templates/common/etc/systemd/system/kubelet.service b/promenade/templates/common/etc/systemd/system/kubelet.service deleted file mode 100644 index 81dfc3e0..00000000 --- a/promenade/templates/common/etc/systemd/system/kubelet.service +++ /dev/null @@ -1,24 +0,0 @@ -[Unit] -Description=Kubernetes Kubelet -Documentation=https://kubernetes.io/docs/admin/kubelet/ - -[Service] -ExecStart=/usr/local/bin/kubelet \ - --allow-privileged=true \ - --cluster-dns={{ config['Network']['cluster_dns'] }} \ - --cluster-domain={{ config['Network']['cluster_domain'] }} \ - --cni-bin-dir=/opt/cni/bin \ - --cni-conf-dir=/etc/cni/net.d \ - --hostname-override={{ config['Node']['hostname'] }} \ - --kubeconfig=/etc/kubernetes/kubelet/kubeconfig.yaml \ - --network-plugin=cni \ - --node-ip={{ config['Node']['ip'] }} \ - --node-labels={{ config['Node']['labels'] | join(',') }} \ - --pod-manifest-path=/etc/kubernetes/kubelet/manifests \ - --require-kubeconfig=true \ - --v=5 -Restart=on-failure -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/promenade/templates/common/usr/local/bin/bootstrap b/promenade/templates/common/usr/local/bin/bootstrap deleted file mode 100755 index fe65758c..00000000 --- a/promenade/templates/common/usr/local/bin/bootstrap +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -ex - -export DEBIAN_FRONTEND=noninteractive - -apt-get install -y --no-install-recommends \ - {%- for package in config['Versions']['additional_packages'] %} - {{ package }} \ - {%- endfor %} - {{ config['Versions']['packages']['dnsmasq'] }} \ - {{ config['Versions']['packages']['socat'] }} - -systemctl daemon-reload -systemctl enable kubelet -systemctl restart kubelet - - -cat < /etc/resolv.conf -options timeout:1 attempts:1 - -nameserver 127.0.0.1 - -{%- for server in config['Network']['dns_servers'] %} -nameserver {{ server }} -{%- endfor %} -EOF diff --git a/promenade/templates/generate/up.sh b/promenade/templates/generate/up.sh deleted file mode 100644 index 2b26905b..00000000 --- a/promenade/templates/generate/up.sh +++ /dev/null @@ -1 +0,0 @@ -{% include "up.sh" with context %} diff --git a/promenade/templates/generate/validate-bootstrap.sh b/promenade/templates/generate/validate-bootstrap.sh deleted file mode 100644 index 41cea1ca..00000000 --- a/promenade/templates/generate/validate-bootstrap.sh +++ /dev/null @@ -1,5 +0,0 @@ -{% include "common_validation.sh" with context %} - -wait_for_ready_nodes 1 - -validate_kubectl_logs diff --git a/promenade/templates/generate/validate-cluster.sh b/promenade/templates/generate/validate-cluster.sh deleted file mode 100644 index 1d5dbd37..00000000 --- a/promenade/templates/generate/validate-cluster.sh +++ /dev/null @@ -1,6 +0,0 @@ -{% include "common_validation.sh" with context %} - -EXPECTED_NODE_COUNT={{ config['Cluster']['nodes'] | length }} -wait_for_ready_nodes $EXPECTED_NODE_COUNT - -validate_kubectl_logs diff --git a/promenade/templates/genesis/etc/kubernetes/armada-loader/assets/external-armada.yaml b/promenade/templates/genesis/etc/kubernetes/armada-loader/assets/external-armada.yaml deleted file mode 100644 index 83a6132e..00000000 --- a/promenade/templates/genesis/etc/kubernetes/armada-loader/assets/external-armada.yaml +++ /dev/null @@ -1 +0,0 @@ -{{ config.get_armada_documents() | yaml_safe_dump_all }} diff --git a/promenade/templates/genesis/etc/kubernetes/armada-loader/assets/promenade-armada.yaml b/promenade/templates/genesis/etc/kubernetes/armada-loader/assets/promenade-armada.yaml deleted file mode 100644 index e8acb664..00000000 --- a/promenade/templates/genesis/etc/kubernetes/armada-loader/assets/promenade-armada.yaml +++ /dev/null @@ -1,101 +0,0 @@ ---- -schema: armada/Manifest/v1 -metadata: - schema: metadata/Document/v1 - name: promenade-networking -data: - release_prefix: promenade - chart_groups: - - promenade-networking - - promenade-rbac ---- -schema: armada/ChartGroup/v1 -metadata: - schema: metadata/Document/v1 - name: promenade-networking -data: - description: core - sequenced: True - chart_group: - - calico - - kube-dns ---- -schema: armada/ChartGroup/v1 -metadata: - schema: metadata/Document/v1 - name: promenade-rbac -data: - description: core - chart_group: - - rbac ---- -schema: armada/Chart/v1 -metadata: - schema: metadata/Document/v1 - name: calico -data: - chart_name: calico - release: calico - namespace: kube-system - timeout: 600 - values: - calico: - {%- if config['Network']['calico_interface'] is defined %} - interface: {{ config['Network']['calico_interface'] }} - {%- endif %} - etcd: - credentials: - ca: |- -{{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] | indent(12, True) }} - cert: |- -{{ config.get(kind='Certificate', alias='calico-etcd-node-client')['data'] | indent(12, True) }} - key: |- -{{ config.get(kind='CertificateKey', alias='calico-etcd-node-client')['data'] | indent(12, True) }} - source: - type: local - location: /etc/kubernetes/armada-loader/assets/charts - subpath: calico - dependencies: [] ---- -schema: armada/Chart/v1 -metadata: - schema: metadata/Document/v1 - name: kube-dns -data: - chart_name: kube-dns - release: kube-dns - namespace: kube-system - timeout: 600 - values: - kube_dns: - replicas: 1 - {%- if config['Network']['dns_servers'] is defined %} - upstream_nameservers: - {%- for nameserver in config['Network']['dns_servers'] %} - - {{ nameserver }} - {%- endfor %} - {%- else %} - upstream_nameservers: [] - {%- endif %} - source: - type: local - location: /etc/kubernetes/armada-loader/assets/charts - subpath: kube-dns - dependencies: [] ---- -schema: armada/Chart/v1 -metadata: - schema: metadata/Document/v1 - name: rbac -data: - chart_name: rbac - release: rbac - namespace: kube-system - timeout: 600 - values: {} - source: - type: local - location: /etc/kubernetes/armada-loader/assets/charts - subpath: rbac - dependencies: [] -... diff --git a/promenade/templates/genesis/etc/kubernetes/armada-loader/config b/promenade/templates/genesis/etc/kubernetes/armada-loader/config deleted file mode 100644 index 5e2b8d4c..00000000 --- a/promenade/templates/genesis/etc/kubernetes/armada-loader/config +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: v1 -clusters: -- cluster: - server: https://kubernetes.default.svc.{{ config['Network']['cluster_domain'] }} - certificate-authority: pki/cluster-ca.pem - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: armada-loader - name: armada-loader@kubernetes -current-context: armada-loader@kubernetes -kind: Config -preferences: {} -users: -- name: armada-loader - user: - client-certificate: pki/armada-loader.pem - client-key: pki/armada-loader-key.pem diff --git a/promenade/templates/genesis/etc/kubernetes/armada-loader/pki/armada-loader-key.pem b/promenade/templates/genesis/etc/kubernetes/armada-loader/pki/armada-loader-key.pem deleted file mode 100644 index a9dfb2dd..00000000 --- a/promenade/templates/genesis/etc/kubernetes/armada-loader/pki/armada-loader-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', name='admin')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/armada-loader/pki/armada-loader.pem b/promenade/templates/genesis/etc/kubernetes/armada-loader/pki/armada-loader.pem deleted file mode 100644 index 3cfb3dc9..00000000 --- a/promenade/templates/genesis/etc/kubernetes/armada-loader/pki/armada-loader.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', name='admin')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/armada-loader/pki/cluster-ca.pem b/promenade/templates/genesis/etc/kubernetes/armada-loader/pki/cluster-ca.pem deleted file mode 100644 index 51adb572..00000000 --- a/promenade/templates/genesis/etc/kubernetes/armada-loader/pki/cluster-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='cluster')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/asset-loader/assets/configmap-promenade-join.yaml b/promenade/templates/genesis/etc/kubernetes/asset-loader/assets/configmap-promenade-join.yaml deleted file mode 100644 index 6439b305..00000000 --- a/promenade/templates/genesis/etc/kubernetes/asset-loader/assets/configmap-promenade-join.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ucp -spec: {} - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: promenade-join-sh - namespace: ucp -data: - join.sh: |- -{%- filter indent(4, True) %} -{% include "up.sh" %} -{%- endfilter %} diff --git a/promenade/templates/genesis/etc/kubernetes/asset-loader/kubeconfig.yaml b/promenade/templates/genesis/etc/kubernetes/asset-loader/kubeconfig.yaml deleted file mode 100644 index ff738781..00000000 --- a/promenade/templates/genesis/etc/kubernetes/asset-loader/kubeconfig.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: v1 -clusters: -- cluster: - server: https://kubernetes:6443 - certificate-authority: /etc/kubernetes/asset-loader/pki/cluster-ca.pem - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: asset-loader - name: asset-loader@kubernetes -current-context: asset-loader@kubernetes -kind: Config -preferences: {} -users: -- name: asset-loader - user: - client-certificate: /etc/kubernetes/asset-loader/pki/asset-loader.pem - client-key: /etc/kubernetes/asset-loader/pki/asset-loader-key.pem diff --git a/promenade/templates/genesis/etc/kubernetes/asset-loader/pki/asset-loader-key.pem b/promenade/templates/genesis/etc/kubernetes/asset-loader/pki/asset-loader-key.pem deleted file mode 100644 index a9dfb2dd..00000000 --- a/promenade/templates/genesis/etc/kubernetes/asset-loader/pki/asset-loader-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', name='admin')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/asset-loader/pki/asset-loader.pem b/promenade/templates/genesis/etc/kubernetes/asset-loader/pki/asset-loader.pem deleted file mode 100644 index 3cfb3dc9..00000000 --- a/promenade/templates/genesis/etc/kubernetes/asset-loader/pki/asset-loader.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', name='admin')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/asset-loader/pki/cluster-ca.pem b/promenade/templates/genesis/etc/kubernetes/asset-loader/pki/cluster-ca.pem deleted file mode 100644 index 51adb572..00000000 --- a/promenade/templates/genesis/etc/kubernetes/asset-loader/pki/cluster-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='cluster')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/client-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/client-ca.pem deleted file mode 100644 index 7491e20d..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/client-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client-key.pem deleted file mode 100644 index 29b4f84e..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-0-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client.pem deleted file mode 100644 index d0f2c764..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-0-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer-key.pem deleted file mode 100644 index 61b19257..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-0-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer.pem deleted file mode 100644 index beab792a..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-0-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/peer-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/peer-ca.pem deleted file mode 100644 index 81c2a591..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/peer-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='calico-etcd-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/client-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/client-ca.pem deleted file mode 100644 index 7491e20d..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/client-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client-key.pem deleted file mode 100644 index cb44de87..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-1-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client.pem deleted file mode 100644 index 90e1ce30..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-1-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer-key.pem deleted file mode 100644 index e249cd95..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-1-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer.pem deleted file mode 100644 index a1fb1228..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-1-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/peer-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/peer-ca.pem deleted file mode 100644 index 81c2a591..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/peer-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='calico-etcd-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/client-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/client-ca.pem deleted file mode 100644 index 6f6fbed3..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/client-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='etcd-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-client-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-client-key.pem deleted file mode 100644 index 9073d56b..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-client-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', name='auxiliary-etcd-0-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-client.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-client.pem deleted file mode 100644 index ebd8c5e7..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-client.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', name='auxiliary-etcd-0-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-peer-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-peer-key.pem deleted file mode 100644 index b5ce1958..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-peer-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', name='auxiliary-etcd-0-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-peer.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-peer.pem deleted file mode 100644 index 92076936..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/etcd-peer.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', name='auxiliary-etcd-0-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/peer-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/peer-ca.pem deleted file mode 100644 index 04003625..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-0/pki/peer-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='etcd-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/client-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/client-ca.pem deleted file mode 100644 index 6f6fbed3..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/client-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='etcd-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-client-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-client-key.pem deleted file mode 100644 index 65821b61..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-client-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', name='auxiliary-etcd-1-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-client.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-client.pem deleted file mode 100644 index 602dcd29..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-client.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', name='auxiliary-etcd-1-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-peer-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-peer-key.pem deleted file mode 100644 index 3ebf84f7..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-peer-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', name='auxiliary-etcd-1-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-peer.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-peer.pem deleted file mode 100644 index 66b11820..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/etcd-peer.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', name='auxiliary-etcd-1-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/peer-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/peer-ca.pem deleted file mode 100644 index 04003625..00000000 --- a/promenade/templates/genesis/etc/kubernetes/auxiliary-etcd-1/pki/peer-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='etcd-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/genesis/kubeconfig.yaml b/promenade/templates/genesis/etc/kubernetes/genesis/kubeconfig.yaml deleted file mode 100644 index 299c61b8..00000000 --- a/promenade/templates/genesis/etc/kubernetes/genesis/kubeconfig.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: v1 -clusters: -- cluster: - server: https://127.0.0.1:6443 - certificate-authority: /target/etc/kubernetes/admin/pki/cluster-ca.pem - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: genesis - name: genesis@kubernetes -current-context: genesis@kubernetes -kind: Config -preferences: {} -users: -- name: genesis - user: - client-certificate: /target/etc/kubernetes/admin/pki/admin.pem - client-key: /target/etc/kubernetes/admin/pki/admin-key.pem diff --git a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/armada-loader.yaml b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/armada-loader.yaml deleted file mode 100644 index d6bbbd81..00000000 --- a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/armada-loader.yaml +++ /dev/null @@ -1,111 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: armada-loader - namespace: kube-system - labels: - app: promenade - component: armada-loader -spec: - hostNetwork: true - containers: - - name: loader - image: {{ config['Versions']['images']['armada'] }} - imagePullPolicy: IfNotPresent - {%- if config['Network']['http_proxy'] is defined - or config['Network']['https_proxy'] is defined - or config['Network']['no_proxy'] is defined %} - env: - {%- endif %} - {%- if config['Network']['http_proxy'] is defined %} - - name: HTTP_PROXY - value: {{ config['Network']['http_proxy'] }} - - name: http_proxy - value: {{ config['Network']['http_proxy'] }} - {%- endif %} - {%- if config['Network']['https_proxy'] is defined %} - - name: HTTPS_PROXY - value: {{ config['Network']['https_proxy'] }} - - name: https_proxy - value: {{ config['Network']['https_proxy'] }} - {%- endif %} - {%- if config['Network']['no_proxy'] is defined %} - - name: NO_PROXY - value: {{ config['Network']['no_proxy'] }} - - name: no_proxy - value: {{ config['Network']['no_proxy'] }} - {%- endif %} - command: - - /bin/bash - - -c - - |- - set -x - - cd /etc/kubernetes/armada-loader/assets - if [ -s promenade-armada.yaml ]; then - while true; do - sleep 10 - if armada --debug apply --tiller-host 127.0.0.1 promenade-armada.yaml ; then - break - fi - done - fi - - if [ -s external-armada.yaml ]; then - while true; do - sleep 10 - if armada --debug apply --tiller-host 127.0.0.1 external-armada.yaml ; then - break - fi - done - fi - - touch /shared/armada.done - - # Sleep so that kubelet doesn't restart this pod before it kills it - sleep 10000 - - volumeMounts: - - name: config - mountPath: /etc/kubernetes/armada-loader - readOnly: true - - name: config - mountPath: /armada/.kube - readOnly: true - - name: shared - mountPath: /shared - - - name: monitor - image: busybox:latest - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -c - - |- - set -x - - while [ ! -f /shared/armada.done ] ; do - sleep 1 - done - - rm -f /etc/kubernetes/kubelet/manifests/armada-loader.yaml - - # Sleep so that kubelet doesn't restart this pod before it kills it - sleep 10000 - - volumeMounts: - - name: shared - mountPath: /shared - - name: manifests - mountPath: /etc/kubernetes/kubelet/manifests - - volumes: - - name: config - hostPath: - path: /etc/kubernetes/armada-loader - - name: manifests - hostPath: - path: /etc/kubernetes/kubelet/manifests - - name: shared - emptyDir: {} diff --git a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/asset-loader.yaml b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/asset-loader.yaml deleted file mode 100644 index 0d5b47ab..00000000 --- a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/asset-loader.yaml +++ /dev/null @@ -1,44 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: asset-loader - namespace: kube-system - labels: - app: promenade - component: asset-loader -spec: - dnsPolicy: Default # Don't use cluster DNS. - hostNetwork: true - containers: - - name: loader - image: {{ config['Versions']['images']['kubernetes']['kubectl'] }} - command: - - /bin/bash - - -c - - |- - set -x - - export KUBECONFIG=/etc/kubernetes/asset-loader/kubeconfig.yaml - DELAY=20 - - while ! /kubectl apply -f /etc/kubernetes/asset-loader/assets; do - sleep $DELAY - done - - rm -rf /etc/kubernetes/kubelet/manifests/asset-loader.yaml - # Sleep so that kubelet doesn't restart this pod before it kills it - sleep 10000 - volumeMounts: - - name: config - mountPath: /etc/kubernetes/asset-loader - readOnly: true - - name: manifests - mountPath: /etc/kubernetes/kubelet/manifests - volumes: - - name: config - hostPath: - path: /etc/kubernetes/asset-loader - - name: manifests - hostPath: - path: /etc/kubernetes/kubelet/manifests diff --git a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml deleted file mode 100644 index dc438eec..00000000 --- a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml +++ /dev/null @@ -1,198 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: auxiliary-calico-etcd - namespace: kube-system - labels: - component: auxiliary-calico-etcd - promenade: genesis -spec: - hostNetwork: true - containers: - - name: auxiliary-calico-etcd-0 - image: {{ config['Versions']['images']['calico']['etcd'] }} - env: - - name: ETCD_NAME - value: auxiliary-calico-etcd-0 - - name: ETCD_STRICT_RECONFIG_CHECK - value: "false" - - name: ETCD_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_PEER_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_DATA_DIR - value: /var/lib/auxiliary-calico-etcd-0 - - name: ETCD_TRUSTED_CA_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/client-ca.pem - - name: ETCD_CERT_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client.pem - - name: ETCD_KEY_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client-key.pem - - name: ETCD_PEER_TRUSTED_CA_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/peer-ca.pem - - name: ETCD_PEER_CERT_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer.pem - - name: ETCD_PEER_KEY_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer-key.pem - - name: ETCD_ADVERTISE_CLIENT_URLS - value: https://{{ config['Node']['hostname'] }}:16666 - - name: ETCD_INITIAL_ADVERTISE_PEER_URLS - value: https://{{ config['Node']['hostname'] }}:16667 - - name: ETCD_INITIAL_CLUSTER_TOKEN - value: promenade-calico-etcd-token - - name: ETCD_LISTEN_CLIENT_URLS - value: https://0.0.0.0:16666 - - name: ETCD_LISTEN_PEER_URLS - value: https://0.0.0.0:16667 - - name: ETCD_INITIAL_CLUSTER_STATE - value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster_state'] }} - - name: ETCD_INITIAL_CLUSTER - value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster'] | join(',') }} - ports: - - name: client - containerPort: 16666 - - name: peer - containerPort: 16667 - resources: - limits: - cpu: 100m - requests: - cpu: 100m - volumeMounts: - - name: data-0 - mountPath: /var/lib/auxiliary-calico-etcd-0 - - name: pki-0 - mountPath: /etc/kubernetes/auxiliary-calico-etcd-0/pki - readOnly: true - - name: auxiliary-calico-etcd-1 - image: {{ config['Versions']['images']['calico']['etcd'] }} - env: - - name: ETCD_NAME - value: auxiliary-calico-etcd-1 - - name: ETCD_STRICT_RECONFIG_CHECK - value: "false" - - name: ETCD_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_PEER_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_DATA_DIR - value: /var/lib/auxiliary-calico-etcd-1 - - name: ETCD_TRUSTED_CA_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/client-ca.pem - - name: ETCD_CERT_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client.pem - - name: ETCD_KEY_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client-key.pem - - name: ETCD_PEER_TRUSTED_CA_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/peer-ca.pem - - name: ETCD_PEER_CERT_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer.pem - - name: ETCD_PEER_KEY_FILE - value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer-key.pem - - name: ETCD_ADVERTISE_CLIENT_URLS - value: https://{{ config['Node']['hostname'] }}:26666 - - name: ETCD_INITIAL_ADVERTISE_PEER_URLS - value: https://{{ config['Node']['hostname'] }}:26667 - - name: ETCD_INITIAL_CLUSTER_TOKEN - value: promenade-calico-etcd-token - - name: ETCD_LISTEN_CLIENT_URLS - value: https://0.0.0.0:26666 - - name: ETCD_LISTEN_PEER_URLS - value: https://0.0.0.0:26667 - - name: ETCD_INITIAL_CLUSTER_STATE - value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster_state'] }} - - name: ETCD_INITIAL_CLUSTER - value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster'] | join(',') }} - ports: - - name: client - containerPort: 26666 - - name: peer - containerPort: 26667 - resources: - limits: - cpu: 100m - requests: - cpu: 100m - volumeMounts: - - name: data-1 - mountPath: /var/lib/auxiliary-calico-etcd-1 - - name: pki-1 - mountPath: /etc/kubernetes/auxiliary-calico-etcd-1/pki - readOnly: true - - name: cluster-monitor - image: {{ config['Versions']['images']['calico']['etcd'] }} - command: - - sh - - -c - - |- - set -x - while true; do - if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -ge {{ config['Masters']['nodes'] | length }} ]; then - {%- for master in config['Masters']['nodes'] %} - etcdctl member add {{ master['hostname'] }} --peer-urls https://{{ master['hostname'] }}:6667 - {%- endfor %} - break - fi - done - while true; do - sleep 5 - if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -eq {{ 2 + (config['Masters']['nodes'] | length) }} ]; then - etcdctl member remove $(etcdctl member list | grep auxiliary-calico-etcd-1 | cut -d , -f 1) - etcdctl member remove $(etcdctl member list | grep auxiliary-calico-etcd-0 | cut -d , -f 1) - sleep 60 - rm -rf \ - /var/lib/auxiliary-calico-etcd-0 \ - /var/lib/auxiliary-calico-etcd-1 \ - /etc/kubernetes/auxiliary-calico-etcd-0 \ - /etc/kubernetes/auxiliary-calico-etcd-1 \ - /etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml - sleep 10000 - fi - done - resources: - limits: - cpu: 100m - requests: - cpu: 100m - env: - - name: ETCDCTL_API - value: "3" - - name: ETCDCTL_CACERT - value: /etc/kubernetes/calico-etcd/pki/client-ca.pem - - name: ETCDCTL_CERT - value: /etc/kubernetes/calico-etcd/pki/etcd-client.pem - - name: ETCDCTL_ENDPOINTS - value: https://{{ config['Node']['ip'] }}:6666 - - name: ETCDCTL_KEY - value: /etc/kubernetes/calico-etcd/pki/etcd-client-key.pem - volumeMounts: - - name: pki - mountPath: /etc/kubernetes/calico-etcd/pki - readOnly: true - - name: manifests - mountPath: /etc/kubernetes/kubelet/manifests - - name: varlib - mountPath: /var/lib - volumes: - - name: data-0 - hostPath: - path: /var/lib/auxiliary-calico-etcd-0 - - name: data-1 - hostPath: - path: /var/lib/auxiliary-calico-etcd-1 - - name: pki - hostPath: - path: /etc/kubernetes/calico-etcd/pki - - name: pki-0 - hostPath: - path: /etc/kubernetes/auxiliary-calico-etcd-0/pki - - name: pki-1 - hostPath: - path: /etc/kubernetes/auxiliary-calico-etcd-1/pki - - name: manifests - hostPath: - path: /etc/kubernetes/kubelet/manifests - - name: varlib - hostPath: - path: /var/lib diff --git a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml deleted file mode 100644 index 3374b9bb..00000000 --- a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml +++ /dev/null @@ -1,198 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: auxiliary-etcd - namespace: kube-system - labels: - component: auxiliary-etcd - promenade: genesis -spec: - hostNetwork: true - containers: - - name: auxiliary-etcd-0 - image: {{ config['Versions']['images']['kubernetes']['etcd'] }} - env: - - name: ETCD_NAME - value: auxiliary-etcd-0 - - name: ETCD_STRICT_RECONFIG_CHECK - value: "false" - - name: ETCD_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_PEER_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_DATA_DIR - value: /var/lib/auxiliary-etcd-0 - - name: ETCD_TRUSTED_CA_FILE - value: /etc/kubernetes/auxiliary-etcd-0/pki/client-ca.pem - - name: ETCD_CERT_FILE - value: /etc/kubernetes/auxiliary-etcd-0/pki/etcd-client.pem - - name: ETCD_KEY_FILE - value: /etc/kubernetes/auxiliary-etcd-0/pki/etcd-client-key.pem - - name: ETCD_PEER_TRUSTED_CA_FILE - value: /etc/kubernetes/auxiliary-etcd-0/pki/peer-ca.pem - - name: ETCD_PEER_CERT_FILE - value: /etc/kubernetes/auxiliary-etcd-0/pki/etcd-peer.pem - - name: ETCD_PEER_KEY_FILE - value: /etc/kubernetes/auxiliary-etcd-0/pki/etcd-peer-key.pem - - name: ETCD_ADVERTISE_CLIENT_URLS - value: https://{{ config['Node']['hostname'] }}:12379 - - name: ETCD_INITIAL_ADVERTISE_PEER_URLS - value: https://{{ config['Node']['hostname'] }}:12380 - - name: ETCD_INITIAL_CLUSTER_TOKEN - value: promenade-kube-etcd-token - - name: ETCD_LISTEN_CLIENT_URLS - value: https://0.0.0.0:12379 - - name: ETCD_LISTEN_PEER_URLS - value: https://0.0.0.0:12380 - - name: ETCD_INITIAL_CLUSTER_STATE - value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster_state'] }} - - name: ETCD_INITIAL_CLUSTER - value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster'] | join(',') }} - ports: - - name: client - containerPort: 12379 - - name: peer - containerPort: 12380 - resources: - limits: - cpu: 100m - requests: - cpu: 100m - volumeMounts: - - name: data-0 - mountPath: /var/lib/auxiliary-etcd-0 - - name: pki-0 - mountPath: /etc/kubernetes/auxiliary-etcd-0/pki - readOnly: true - - name: auxiliary-etcd-1 - image: {{ config['Versions']['images']['kubernetes']['etcd'] }} - env: - - name: ETCD_NAME - value: auxiliary-etcd-1 - - name: ETCD_STRICT_RECONFIG_CHECK - value: "false" - - name: ETCD_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_PEER_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_DATA_DIR - value: /var/lib/auxiliary-etcd-1 - - name: ETCD_TRUSTED_CA_FILE - value: /etc/kubernetes/auxiliary-etcd-1/pki/client-ca.pem - - name: ETCD_CERT_FILE - value: /etc/kubernetes/auxiliary-etcd-1/pki/etcd-client.pem - - name: ETCD_KEY_FILE - value: /etc/kubernetes/auxiliary-etcd-1/pki/etcd-client-key.pem - - name: ETCD_PEER_TRUSTED_CA_FILE - value: /etc/kubernetes/auxiliary-etcd-1/pki/peer-ca.pem - - name: ETCD_PEER_CERT_FILE - value: /etc/kubernetes/auxiliary-etcd-1/pki/etcd-peer.pem - - name: ETCD_PEER_KEY_FILE - value: /etc/kubernetes/auxiliary-etcd-1/pki/etcd-peer-key.pem - - name: ETCD_ADVERTISE_CLIENT_URLS - value: https://{{ config['Node']['hostname'] }}:22379 - - name: ETCD_INITIAL_ADVERTISE_PEER_URLS - value: https://{{ config['Node']['hostname'] }}:22380 - - name: ETCD_INITIAL_CLUSTER_TOKEN - value: promenade-kube-etcd-token - - name: ETCD_LISTEN_CLIENT_URLS - value: https://0.0.0.0:22379 - - name: ETCD_LISTEN_PEER_URLS - value: https://0.0.0.0:22380 - - name: ETCD_INITIAL_CLUSTER_STATE - value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster_state'] }} - - name: ETCD_INITIAL_CLUSTER - value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster'] | join(',') }} - ports: - - name: client - containerPort: 22379 - - name: peer - containerPort: 22380 - resources: - limits: - cpu: 100m - requests: - cpu: 100m - volumeMounts: - - name: data-1 - mountPath: /var/lib/auxiliary-etcd-1 - - name: pki-1 - mountPath: /etc/kubernetes/auxiliary-etcd-1/pki - readOnly: true - - name: cluster-monitor - image: {{ config['Versions']['images']['kubernetes']['etcd'] }} - command: - - sh - - -c - - |- - set -x - while true; do - if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -ge {{ config['Masters']['nodes'] | length }} ]; then - {%- for master in config['Masters']['nodes'] %} - etcdctl member add {{ master['hostname'] }} --peer-urls https://{{ master['hostname'] }}:2380 - {%- endfor %} - break - fi - done - while true; do - sleep 5 - if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -eq {{ 2 + (config['Masters']['nodes'] | length) }} ]; then - etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-1 | cut -d , -f 1) - etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-0 | cut -d , -f 1) - sleep 60 - rm -rf \ - /var/lib/auxiliary-etcd-0 \ - /var/lib/auxiliary-etcd-1 \ - /etc/kubernetes/auxiliary-etcd-0 \ - /etc/kubernetes/auxiliary-etcd-1 \ - /etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml - sleep 10000 - fi - done - resources: - limits: - cpu: 100m - requests: - cpu: 100m - env: - - name: ETCDCTL_API - value: "3" - - name: ETCDCTL_CACERT - value: /etc/kubernetes/etcd/pki/client-ca.pem - - name: ETCDCTL_CERT - value: /etc/kubernetes/etcd/pki/etcd-client.pem - - name: ETCDCTL_ENDPOINTS - value: https://{{ config['Node']['ip'] }}:2379 - - name: ETCDCTL_KEY - value: /etc/kubernetes/etcd/pki/etcd-client-key.pem - volumeMounts: - - name: pki - mountPath: /etc/kubernetes/etcd/pki - readOnly: true - - name: manifests - mountPath: /etc/kubernetes/kubelet/manifests - - name: varlib - mountPath: /var/lib - volumes: - - name: data-0 - hostPath: - path: /var/lib/auxiliary-etcd-0 - - name: data-1 - hostPath: - path: /var/lib/auxiliary-etcd-1 - - name: pki - hostPath: - path: /etc/kubernetes/etcd/pki - - name: pki-0 - hostPath: - path: /etc/kubernetes/auxiliary-etcd-0/pki - - name: pki-1 - hostPath: - path: /etc/kubernetes/auxiliary-etcd-1/pki - - name: manifests - hostPath: - path: /etc/kubernetes/kubelet/manifests - - name: varlib - hostPath: - path: /var/lib diff --git a/promenade/templates/include/cleanup.sh b/promenade/templates/include/cleanup.sh new file mode 100644 index 00000000..206254aa --- /dev/null +++ b/promenade/templates/include/cleanup.sh @@ -0,0 +1,5 @@ +set +x +log +log === Restarting kubelet === +set -x +systemctl restart kubelet diff --git a/promenade/templates/include/common_validation.sh b/promenade/templates/include/common_validation.sh deleted file mode 100644 index 797b9907..00000000 --- a/promenade/templates/include/common_validation.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env bash - -if [ "$(id -u)" != "0" ]; then - echo "This script must be run as root." 1>&2 - exit 1 -fi - -set -ex - -export KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml - -function log { - echo $* 1>&2 -} - -function report_docker_exited_containers { - for container_id in $(docker ps -q --filter "status=exited"); do - log Report for exited container $container_id - docker inspect $container_id - docker logs $container_id - done -} - -function report_docker_state { - log General docker state report - docker info - docker ps -a - report_docker_exited_containers -} - -function report_kube_state { - log General cluster state report - kubectl get nodes 1>&2 - kubectl get --all-namespaces pods -o wide 1>&2 -} - -function fail { - report_docker_state - report_kube_state - exit 1 -} - -function wait_for_ready_nodes { - set +x - - NODES=$1 - SECONDS=${2:-600} - - log $(date) Waiting $SECONDS seconds for $NODES Ready nodes. - - NODE_READY_JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[?(@.type=="Ready")]}{@.type}={@.status}{"\n"}{end}{end}' - - end=$(($(date +%s) + $SECONDS)) - while true; do - READY_NODE_COUNT=$(kubectl get nodes -o jsonpath="${NODE_READY_JSONPATH}" | grep "Ready=True" | wc -l) - if [ $NODES -ne $READY_NODE_COUNT ]; then - now=$(date +%s) - if [ $now -gt $end ]; then - log $(date) Nodes were not all ready before timeout. - fail - fi - sleep 5 - else - log $(date) Found expected nodes. - break - fi - done - - set -x -} - -function wait_for_pod_termination { - set +x - - NAMESPACE=$1 - POD_NAME=$2 - SECONDS=${3:-120} - - log $(date) Waiting $SECONDS seconds for termination of pod $POD_NAME - - POD_PHASE_JSONPATH='{.status.phase}' - - end=$(($(date +%s) + $SECONDS)) - while true; do - POD_PHASE=$(kubectl --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME) - if [ "x$POD_PHASE" = "xSucceeded" ]; then - log $(date) Pod $POD_NAME succeeded. - break - elif [ "x$POD_PHASE" = "xFailed" ]; then - log $(date) Pod $POD_NAME failed. - kubectl --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 - fail - else - now=$(date +%s) - if [ $now -gt $end ]; then - log $(date) Pod did not terminate before timeout. - kubectl --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 - fail - fi - sleep 1 - fi - done - - set -x -} - -function validate_kubectl_logs { - NAMESPACE=default - POD_NAME=log-test-$(date +%s) - - cat <&2 - fail - fi -} diff --git a/promenade/templates/include/genesis-etcd/common-volumes.yaml b/promenade/templates/include/genesis-etcd/common-volumes.yaml new file mode 100644 index 00000000..a098fa66 --- /dev/null +++ b/promenade/templates/include/genesis-etcd/common-volumes.yaml @@ -0,0 +1,6 @@ + - name: data + hostPath: + path: /var/lib/etcd/{{ etcd_name }} + - name: pki + hostPath: + path: /etc/genesis/etcd/pki diff --git a/promenade/templates/include/genesis-etcd/server-container.yaml b/promenade/templates/include/genesis-etcd/server-container.yaml new file mode 100644 index 00000000..f6ba4567 --- /dev/null +++ b/promenade/templates/include/genesis-etcd/server-container.yaml @@ -0,0 +1,63 @@ + - name: etcd + image: {{ config['Genesis:images.kubernetes.etcd'] }} + env: + - name: ETCD_NAME + value: {{ etcd_name }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: ETCD_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_PEER_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_DATA_DIR + value: /var/lib/etcd + - name: ETCD_TRUSTED_CA_FILE + value: /etc/etcd/pki/client-ca.pem + - name: ETCD_CERT_FILE + value: /etc/etcd/pki/etcd-client.pem + - name: ETCD_KEY_FILE + value: /etc/etcd/pki/etcd-client-key.pem + - name: ETCD_PEER_TRUSTED_CA_FILE + value: /etc/etcd/pki/peer-ca.pem + - name: ETCD_PEER_CERT_FILE + value: /etc/etcd/pki/etcd-peer.pem + - name: ETCD_PEER_KEY_FILE + value: /etc/etcd/pki/etcd-peer-key.pem + - name: ETCD_ADVERTISE_CLIENT_URLS + value: https://$(POD_IP):{{ client_port }} + - name: ETCD_INITIAL_ADVERTISE_PEER_URLS + value: https://$(POD_IP):{{ peer_port }} + - name: ETCD_INITIAL_CLUSTER_TOKEN + value: promenade-kube-etcd-token + - name: ETCD_LISTEN_CLIENT_URLS + value: https://0.0.0.0:{{ client_port }} + - name: ETCD_LISTEN_PEER_URLS + value: https://0.0.0.0:{{ peer_port }} + - name: ETCD_INITIAL_CLUSTER_STATE + value: new + - name: ETCD_INITIAL_CLUSTER + value: genesis=https://{{ config['Genesis:ip'] }}:2380,auxiliary=https://{{ config['Genesis:ip'] }}:12380 + - name: ETCDCTL_API + value: '3' + - name: ETCDCTL_DIAL_TIMEOUT + value: 3s + - name: ETCDCTL_ENDPOINTS + value: https://$(POD_IP):{{ client_port }},https://127.0.0.1:{{ client_port }} + - name: ETCDCTL_CACERT + value: $(ETCD_TRUSTED_CA_FILE) + - name: ETCDCTL_CERT + value: $(ETCD_CERT_FILE) + - name: ETCDCTL_KEY + value: $(ETCD_KEY_FILE) + ports: + - name: client + containerPort: {{ client_port }} + - name: peer + containerPort: {{ peer_port }} + volumeMounts: + - name: data + mountPath: /var/lib/etcd + - name: pki + mountPath: /etc/etcd/pki diff --git a/promenade/templates/include/header.sh b/promenade/templates/include/header.sh new file mode 100644 index 00000000..2a467a03 --- /dev/null +++ b/promenade/templates/include/header.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -e + +{% include "utils.sh" with context %} + +# Ensure the script is running as root. +# +if [ "$(id -u)" != "0" ]; then + echo "This script must be run as root." 1>&2 + exit 1 +fi + +function log { + echo $(date) $* 1>&2 +} diff --git a/promenade/templates/include/up.sh b/promenade/templates/include/up.sh index 79963e31..573f8656 100644 --- a/promenade/templates/include/up.sh +++ b/promenade/templates/include/up.sh @@ -1,78 +1,70 @@ -#!/usr/bin/env bash +# Disable overwriting our resolv.conf +# +resolvconf --disable-updates -if [ "$(id -u)" != "0" ]; then - echo "This script must be run as root." 1>&2 - exit 1 -fi +# Unpack prepared files into place +# +set +x +log +log === Extracting prepared files === +echo "{{ tarball | b64enc }}" | base64 -d | tar -zxv -C / | tee /etc/promenade-manifest -if [ "x$1" = "x" ]; then - echo "Path to node configuration required." 1>&2 - exit 1 -fi +# Adding apt repositories +# +set +x +log +log === Adding APT Keys=== +set -x +{%- for key in config.get_path('HostSystem:packages.keys', []) %} +apt-key add - <<"ENDKEY" +{{ key }} +ENDKEY +{%- endfor %} -set -ex +# Disable swap +# +set +x +log +log === Disabling swap === +set -x +swapoff -a +sed --in-place '/\bswap\b/d' /etc/fstab -#Proxy Variables -http_proxy={{ config['Network'].get('http_proxy', '') }} -https_proxy={{ config['Network'].get('https_proxy', '') }} -no_proxy={{ config['Network'].get('no_proxy', '') }} - -DOCKER_HTTP_PROXY=${DOCKER_HTTP_PROXY:-${HTTP_PROXY:-${http_proxy}}} -DOCKER_HTTPS_PROXY=${DOCKER_HTTPS_PROXY:-${HTTPS_PROXY:-${https_proxy}}} -DOCKER_NO_PROXY=${DOCKER_NO_PROXY:-${NO_PROXY:-${no_proxy}}} +# Set proxy variables +# +set +x +log +log === Setting proxy variables === +set -x +export http_proxy={{ config['KubernetesNetwork:proxy.url'] | default('', true) }} +export https_proxy={{ config['KubernetesNetwork:proxy.url'] | default('', true) }} +export no_proxy={{ config.get(kind='KubernetesNetwork') | fill_no_proxy }} -mkdir -p /etc/docker -cat < /etc/docker/daemon.json -{ - "live-restore": true, - "storage-driver": "overlay2" -} -EOS - -#Configuration for Docker Behind a Proxy -mkdir -p /etc/systemd/system/docker.service.d - -#Set HTTPS Proxy Variable -cat < /etc/systemd/system/docker.service.d/http-proxy.conf -[Service] -Environment="HTTP_PROXY=${DOCKER_HTTP_PROXY}" -EOF - -#Set HTTPS Proxy Variable -cat < /etc/systemd/system/docker.service.d/https-proxy.conf -[Service] -Environment="HTTPS_PROXY=${DOCKER_HTTPS_PROXY}" -EOF - -#Set No Proxy Variable -cat < /etc/systemd/system/docker.service.d/no-proxy.conf -[Service] -Environment="NO_PROXY=${DOCKER_NO_PROXY}" -EOF - -#Reload systemd and docker if present -systemctl daemon-reload -systemctl restart docker || true +# Install system packages +# +set +x +log +log === Installing system packages === +set -x export DEBIAN_FRONTEND=noninteractive -apt-get update -qq -apt-get install -y -qq --no-install-recommends \ - {{ config['Versions']['packages']['docker'] }} +apt-get update +apt-get install -y --no-install-recommends \ + {%- for package in config['HostSystem:packages.additional'] | default([]) %} + {{ package }} \ + {%- endfor %} + {{ config['HostSystem:packages.required.docker'] }} \ + {{ config['HostSystem:packages.required.socat'] }} -if [ -f "${PROMENADE_LOAD_IMAGE}" ]; then - echo === Loading updated promenade image === - docker load -i "${PROMENADE_LOAD_IMAGE}" -fi -docker run -t --rm \ - --net host \ - -v /:/target \ - {{ config['Versions']['images']['promenade'] }} \ - promenade \ - -v \ - up \ - --hostname $(hostname) \ - --config-path /target$(realpath $1) 2>&1 - -touch /var/lib/prom.done +# Start core processes +# +set +x +log +log === Starting Docker and Kubelet === +set -x +systemctl daemon-reload +systemctl restart docker || true +systemctl enable kubelet +systemctl restart kubelet diff --git a/promenade/templates/include/utils.sh b/promenade/templates/include/utils.sh new file mode 100644 index 00000000..c0d7d1bc --- /dev/null +++ b/promenade/templates/include/utils.sh @@ -0,0 +1,252 @@ +if [ "$(id -u)" != "0" ]; then + echo "This script must be run as root." 1>&2 + exit 1 +fi + +set -x + +export KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml + +function report_docker_exited_containers { + for container_id in $(docker ps -q --filter "status=exited"); do + log Report for exited container $container_id + docker inspect $container_id + docker logs $container_id + done +} + +function report_docker_state { + log General docker state report + docker info + docker ps -a + report_docker_exited_containers +} + +function report_kube_state { + log General cluster state report + kubectl --request-timeout 15s get nodes 1>&2 + kubectl --request-timeout 15s get --all-namespaces pods -o wide 1>&2 +} + +function fail { + set +e + report_docker_state + report_kube_state + exit 1 +} + +function wait_for_ready_nodes { + set +x + + NODES=$1 + SEC=${2:-600} + + log Waiting $SEC seconds for $NODES Ready nodes. + + NODE_READY_JSONPATH='{.items[*].status.conditions[?(@.type=="Ready")].status}' + + end=$(($(date +%s) + $SEC)) + while true; do + READY_NODE_COUNT=$(kubectl --request-timeout 10s get nodes -o jsonpath="${NODE_READY_JSONPATH}" | tr ' ' '\n' | grep True | wc -l) + if [ $NODES -gt $READY_NODE_COUNT ]; then + now=$(date +%s) + if [ $now -gt $end ]; then + log Nodes were not all ready before timeout. + fail + fi + echo -n . + sleep 5 + else + log Found expected nodes. + break + fi + done + + set -x +} + +function wait_for_node_ready { + set +x + + NODE_NAME=$1 + SEC=${2:-3600} + + log Waiting $SEC seconds for $NODE_NAME to be ready. + + NODE_READY_JSONPATH='{.status.conditions[?(@.type=="Ready")].status}' + + end=$(($(date +%s) + $SEC)) + while true; do + if (kubectl --request-timeout 10s get nodes $NODE_NAME -o jsonpath="${NODE_READY_JSONPATH}" | grep True) ; then + log Node $NODE_NAME is ready. + break + else + now=$(date +%s) + if [ $now -gt $end ]; then + log Node $NODE_NAME was not ready before timeout. + fail + fi + echo -n . + sleep 15 + fi + done + + {%- if config['KubernetesNetwork:dns.bootstrap_validation_checks'] is defined %} + REQUIRED_DNS_LOOKUPS=( + {%- for domain in config['KubernetesNetwork:dns.bootstrap_validation_checks'] %} + {{ domain }} + {%- endfor %} + ) + log Waiting $SEC seconds for specified DNS queries to work: + for DOMAIN in ${REQUIRED_DNS_LOOKUPS[*]}; do + log $DOMAIN + done + + while true; do + SUCCESS=1 + for DOMAIN in ${REQUIRED_DNS_LOOKUPS[*]}; do + if ! (dig $DOMAIN > /dev/null 2> /dev/nul); then + SUCCESS=0 + log Failed to resolve $DOMAIN + break + fi + done + + if [ $SUCCESS -eq 1 ]; then + log Resolved specified queries. + break + fi + now=$(date +%s) + if [ $now -gt $end ]; then + log Could not resolve all required DNS names by timeout on $NODE. + fail + fi + sleep 15 + done + {%- endif %} + + set -x +} + +function wait_for_kubernetes_api { + set +x + + SEC=${1:-3600} + + log Waiting $SEC seconds for API response. + + end=$(($(date +%s) + $SEC)) + while true; do + if kubectl --request-timeout 5s get nodes 2>&1 > /dev/null; then + echo 1>&2 + log Got response from Kubernetes API. + break + else + now=$(date +%s) + if [ $now -gt $end ]; then + log API not returning node list before timeout. + fail + fi + echo -n . 1>&2 + sleep 15 + fi + done + + set -x +} + +function register_labels { + set +x + NODE=$1 + TIMEOUT=$2 + shift 2 + + LABELS="$@" + + end=$(($(date +%s) + $SEC)) + while true; do + if kubectl label node $NODE --overwrite $LABELS ; then + echo 1>&2 + log Applied labels $LABELS on $NODE + break + else + now=$(date +%s) + if [ $now -gt $end ]; then + log Failed to apply labels $LABELS on $NODE + fail + fi + echo -n . 1>&2 + sleep 15 + fi + done + set -x +} + +function wait_for_pod_termination { + set +x + + NAMESPACE=$1 + POD_NAME=$2 + SEC=${3:-120} + + log Waiting $SEC seconds for termination of pod $POD_NAME + + POD_PHASE_JSONPATH='{.status.phase}' + + end=$(($(date +%s) + $SEC)) + while true; do + POD_PHASE=$(kubectl --request-timeout 10s --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME) + if [ "x$POD_PHASE" = "xSucceeded" ]; then + log Pod $POD_NAME succeeded. + break + elif [ "x$POD_PHASE" = "xFailed" ]; then + log Pod $POD_NAME failed. + kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 + fail + else + now=$(date +%s) + if [ $now -gt $end ]; then + log Pod did not terminate before timeout. + kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 + fail + fi + sleep 1 + fi + done + + set -x +} + +function validate_kubectl_logs { + NODE=$1 + NAMESPACE=default + POD_NAME=log-test-${NODE}-$(date +%s) + + cat <&2 + fail + fi +} diff --git a/promenade/templates/master/etc/kubernetes/admin/pki/admin-key.pem b/promenade/templates/master/etc/kubernetes/admin/pki/admin-key.pem deleted file mode 100644 index a9dfb2dd..00000000 --- a/promenade/templates/master/etc/kubernetes/admin/pki/admin-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', name='admin')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/admin/pki/admin.pem b/promenade/templates/master/etc/kubernetes/admin/pki/admin.pem deleted file mode 100644 index 3cfb3dc9..00000000 --- a/promenade/templates/master/etc/kubernetes/admin/pki/admin.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', name='admin')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/admin/pki/cluster-ca.pem b/promenade/templates/master/etc/kubernetes/admin/pki/cluster-ca.pem deleted file mode 100644 index 51adb572..00000000 --- a/promenade/templates/master/etc/kubernetes/admin/pki/cluster-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='cluster')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/apiserver/pki/apiserver-key.pem b/promenade/templates/master/etc/kubernetes/apiserver/pki/apiserver-key.pem deleted file mode 100644 index ae8acae8..00000000 --- a/promenade/templates/master/etc/kubernetes/apiserver/pki/apiserver-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', alias='apiserver')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/apiserver/pki/apiserver.pem b/promenade/templates/master/etc/kubernetes/apiserver/pki/apiserver.pem deleted file mode 100644 index 04e82c2c..00000000 --- a/promenade/templates/master/etc/kubernetes/apiserver/pki/apiserver.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', alias='apiserver')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/apiserver/pki/cluster-ca.pem b/promenade/templates/master/etc/kubernetes/apiserver/pki/cluster-ca.pem deleted file mode 100644 index 51adb572..00000000 --- a/promenade/templates/master/etc/kubernetes/apiserver/pki/cluster-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='cluster')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/apiserver/pki/etcd-client-ca.pem b/promenade/templates/master/etc/kubernetes/apiserver/pki/etcd-client-ca.pem deleted file mode 100644 index 6f6fbed3..00000000 --- a/promenade/templates/master/etc/kubernetes/apiserver/pki/etcd-client-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='etcd-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/apiserver/pki/etcd-client-key.pem b/promenade/templates/master/etc/kubernetes/apiserver/pki/etcd-client-key.pem deleted file mode 100644 index 9f75cf77..00000000 --- a/promenade/templates/master/etc/kubernetes/apiserver/pki/etcd-client-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', alias='etcd-apiserver-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/apiserver/pki/etcd-client.pem b/promenade/templates/master/etc/kubernetes/apiserver/pki/etcd-client.pem deleted file mode 100644 index 42619185..00000000 --- a/promenade/templates/master/etc/kubernetes/apiserver/pki/etcd-client.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', alias='etcd-apiserver-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/apiserver/pki/service-account.pub b/promenade/templates/master/etc/kubernetes/apiserver/pki/service-account.pub deleted file mode 100644 index 30866284..00000000 --- a/promenade/templates/master/etc/kubernetes/apiserver/pki/service-account.pub +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='PublicKey', name='service-account')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/client-ca.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/client-ca.pem deleted file mode 100644 index 7491e20d..00000000 --- a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/client-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client-key.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client-key.pem deleted file mode 100644 index cd8fd9e7..00000000 --- a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', alias='calico-etcd-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client.pem deleted file mode 100644 index 9122ec5d..00000000 --- a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', alias='calico-etcd-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer-key.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer-key.pem deleted file mode 100644 index ce06abe2..00000000 --- a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', alias='calico-etcd-peer')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer.pem deleted file mode 100644 index c7e687fc..00000000 --- a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', alias='calico-etcd-peer')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/peer-ca.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/peer-ca.pem deleted file mode 100644 index 81c2a591..00000000 --- a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/peer-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='calico-etcd-peer')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/controller-manager/pki/cluster-ca-key.pem b/promenade/templates/master/etc/kubernetes/controller-manager/pki/cluster-ca-key.pem deleted file mode 100644 index 58a75ff2..00000000 --- a/promenade/templates/master/etc/kubernetes/controller-manager/pki/cluster-ca-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthorityKey', name='cluster')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/controller-manager/pki/cluster-ca.pem b/promenade/templates/master/etc/kubernetes/controller-manager/pki/cluster-ca.pem deleted file mode 100644 index 51adb572..00000000 --- a/promenade/templates/master/etc/kubernetes/controller-manager/pki/cluster-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='cluster')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/controller-manager/pki/controller-manager-key.pem b/promenade/templates/master/etc/kubernetes/controller-manager/pki/controller-manager-key.pem deleted file mode 100644 index 807fa87f..00000000 --- a/promenade/templates/master/etc/kubernetes/controller-manager/pki/controller-manager-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', alias='controller-manager')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/controller-manager/pki/controller-manager.pem b/promenade/templates/master/etc/kubernetes/controller-manager/pki/controller-manager.pem deleted file mode 100644 index 523bdfcf..00000000 --- a/promenade/templates/master/etc/kubernetes/controller-manager/pki/controller-manager.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', alias='controller-manager')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/controller-manager/pki/service-account.key b/promenade/templates/master/etc/kubernetes/controller-manager/pki/service-account.key deleted file mode 100644 index cc5068aa..00000000 --- a/promenade/templates/master/etc/kubernetes/controller-manager/pki/service-account.key +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='PrivateKey', name='service-account')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/etcd/pki/client-ca.pem b/promenade/templates/master/etc/kubernetes/etcd/pki/client-ca.pem deleted file mode 100644 index 6f6fbed3..00000000 --- a/promenade/templates/master/etc/kubernetes/etcd/pki/client-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='etcd-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-client-key.pem b/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-client-key.pem deleted file mode 100644 index 55efdb25..00000000 --- a/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-client-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', alias='etcd-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-client.pem b/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-client.pem deleted file mode 100644 index 3f39bbf6..00000000 --- a/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-client.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', alias='etcd-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-peer-key.pem b/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-peer-key.pem deleted file mode 100644 index e1b1bdcc..00000000 --- a/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-peer-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', alias='etcd-peer')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-peer.pem b/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-peer.pem deleted file mode 100644 index 88794cbf..00000000 --- a/promenade/templates/master/etc/kubernetes/etcd/pki/etcd-peer.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', alias='etcd-peer')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/etcd/pki/peer-ca.pem b/promenade/templates/master/etc/kubernetes/etcd/pki/peer-ca.pem deleted file mode 100644 index 04003625..00000000 --- a/promenade/templates/master/etc/kubernetes/etcd/pki/peer-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='etcd-peer')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml b/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml deleted file mode 100644 index 3733334d..00000000 --- a/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml +++ /dev/null @@ -1,70 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: calico-etcd - namespace: kube-system - labels: - tier: control-plane - component: calico-etcd -spec: - hostNetwork: true - containers: - - name: k8s-etcd - image: {{ config['Versions']['images']['calico']['etcd'] }} - env: - - name: ETCD_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: ETCD_STRICT_RECONFIG_CHECK - value: "false" - - name: ETCD_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_PEER_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_DATA_DIR - value: /var/lib/calico-etcd - - name: ETCD_TRUSTED_CA_FILE - value: /etc/kubernetes/calico-etcd/pki/client-ca.pem - - name: ETCD_CERT_FILE - value: /etc/kubernetes/calico-etcd/pki/etcd-client.pem - - name: ETCD_KEY_FILE - value: /etc/kubernetes/calico-etcd/pki/etcd-client-key.pem - - name: ETCD_PEER_TRUSTED_CA_FILE - value: /etc/kubernetes/calico-etcd/pki/peer-ca.pem - - name: ETCD_PEER_CERT_FILE - value: /etc/kubernetes/calico-etcd/pki/etcd-peer.pem - - name: ETCD_PEER_KEY_FILE - value: /etc/kubernetes/calico-etcd/pki/etcd-peer-key.pem - - name: ETCD_ADVERTISE_CLIENT_URLS - value: https://$(ETCD_NAME):6666 - - name: ETCD_INITIAL_ADVERTISE_PEER_URLS - value: https://$(ETCD_NAME):6667 - - name: ETCD_INITIAL_CLUSTER_TOKEN - value: promenade-calico-etcd-token - - name: ETCD_LISTEN_CLIENT_URLS - value: https://0.0.0.0:6666 - - name: ETCD_LISTEN_PEER_URLS - value: https://0.0.0.0:6667 - - name: ETCD_INITIAL_CLUSTER_STATE - value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster_state'] }} - - name: ETCD_INITIAL_CLUSTER - value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster'] | join(',') }} - ports: - - name: client - containerPort: 6666 - - name: peer - containerPort: 6667 - volumeMounts: - - name: data - mountPath: /var/lib/calico-etcd - - name: pki - mountPath: /etc/kubernetes/calico-etcd/pki - volumes: - - name: data - hostPath: - path: /var/lib/calico-etcd - - name: pki - hostPath: - path: /etc/kubernetes/calico-etcd/pki diff --git a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-apiserver.yaml b/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-apiserver.yaml deleted file mode 100644 index 006c193e..00000000 --- a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-apiserver.yaml +++ /dev/null @@ -1,48 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: kube-apiserver - namespace: kube-system - labels: - tier: control-plane - component: kube-apiserver - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' -spec: - hostNetwork: true - containers: - - name: kube-apiserver - image: {{ config['Versions']['images']['kubernetes']['apiserver'] }} - command: - - /hyperkube - - apiserver - - --advertise-address={{ config['Node']['ip'] }} - - --authorization-mode=RBAC - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds - - --anonymous-auth=false - - --client-ca-file=/etc/kubernetes/pki/cluster-ca.pem - - --insecure-port=8080 - - --insecure-bind-address=127.0.0.1 - - --bind-address=0.0.0.0 - - --runtime-config=batch/v2alpha1=true - - --secure-port=6443 - - --allow-privileged=true - - --etcd-servers=https://kubernetes:2379 - - --etcd-cafile=/etc/kubernetes/pki/etcd-client-ca.pem - - --etcd-certfile=/etc/kubernetes/pki/etcd-client.pem - - --etcd-keyfile=/etc/kubernetes/pki/etcd-client-key.pem - - --service-cluster-ip-range={{ config['Network']['service_ip_cidr'] }} - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --service-account-key-file=/etc/kubernetes/pki/service-account.pub - - --tls-cert-file=/etc/kubernetes/pki/apiserver.pem - - --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem - - --v=5 - volumeMounts: - - name: config - mountPath: /etc/kubernetes - readOnly: true - volumes: - - name: config - hostPath: - path: /etc/kubernetes/apiserver diff --git a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-etcd.yaml b/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-etcd.yaml deleted file mode 100644 index 586f5c99..00000000 --- a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-etcd.yaml +++ /dev/null @@ -1,70 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: kube-etcd - namespace: kube-system - labels: - tier: control-plane - component: kube-etcd -spec: - hostNetwork: true - containers: - - name: k8s-etcd - image: {{ config['Versions']['images']['kubernetes']['etcd'] }} - env: - - name: ETCD_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: ETCD_STRICT_RECONFIG_CHECK - value: "false" - - name: ETCD_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_PEER_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_DATA_DIR - value: /var/lib/kube-etcd - - name: ETCD_TRUSTED_CA_FILE - value: /etc/kubernetes/etcd/pki/client-ca.pem - - name: ETCD_CERT_FILE - value: /etc/kubernetes/etcd/pki/etcd-client.pem - - name: ETCD_KEY_FILE - value: /etc/kubernetes/etcd/pki/etcd-client-key.pem - - name: ETCD_PEER_TRUSTED_CA_FILE - value: /etc/kubernetes/etcd/pki/peer-ca.pem - - name: ETCD_PEER_CERT_FILE - value: /etc/kubernetes/etcd/pki/etcd-peer.pem - - name: ETCD_PEER_KEY_FILE - value: /etc/kubernetes/etcd/pki/etcd-peer-key.pem - - name: ETCD_ADVERTISE_CLIENT_URLS - value: https://$(ETCD_NAME):2379 - - name: ETCD_INITIAL_ADVERTISE_PEER_URLS - value: https://$(ETCD_NAME):2380 - - name: ETCD_INITIAL_CLUSTER_TOKEN - value: promenade-kube-etcd-token - - name: ETCD_LISTEN_CLIENT_URLS - value: https://0.0.0.0:2379 - - name: ETCD_LISTEN_PEER_URLS - value: https://0.0.0.0:2380 - - name: ETCD_INITIAL_CLUSTER_STATE - value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster_state'] }} - - name: ETCD_INITIAL_CLUSTER - value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster'] | join(',') }} - ports: - - name: client - containerPort: 2379 - - name: peer - containerPort: 2380 - volumeMounts: - - name: data - mountPath: /var/lib/kube-etcd - - name: pki - mountPath: /etc/kubernetes/etcd/pki - volumes: - - name: data - hostPath: - path: /var/lib/kube-etcd - - name: pki - hostPath: - path: /etc/kubernetes/etcd/pki diff --git a/promenade/templates/master/etc/kubernetes/scheduler/pki/cluster-ca.pem b/promenade/templates/master/etc/kubernetes/scheduler/pki/cluster-ca.pem deleted file mode 100644 index 51adb572..00000000 --- a/promenade/templates/master/etc/kubernetes/scheduler/pki/cluster-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateAuthority', name='cluster')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/scheduler/pki/scheduler-key.pem b/promenade/templates/master/etc/kubernetes/scheduler/pki/scheduler-key.pem deleted file mode 100644 index 6b59f0af..00000000 --- a/promenade/templates/master/etc/kubernetes/scheduler/pki/scheduler-key.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='CertificateKey', alias='scheduler')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/scheduler/pki/scheduler.pem b/promenade/templates/master/etc/kubernetes/scheduler/pki/scheduler.pem deleted file mode 100644 index 9fe7e480..00000000 --- a/promenade/templates/master/etc/kubernetes/scheduler/pki/scheduler.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(kind='Certificate', alias='scheduler')['data'] }} diff --git a/promenade/templates/roles/common/etc/apt/apt.conf.d/20-proxy.conf b/promenade/templates/roles/common/etc/apt/apt.conf.d/20-proxy.conf new file mode 100644 index 00000000..d101b7c3 --- /dev/null +++ b/promenade/templates/roles/common/etc/apt/apt.conf.d/20-proxy.conf @@ -0,0 +1,5 @@ +# Generated by Promenade +{% if config['KubernetesNetwork:proxy.url'] is defined -%} +Acquire::http::Proxy "{{ config['KubernetesNetwork:proxy.url'] }}"; +Acquire::https::Proxy "{{ config['KubernetesNetwork:proxy.url'] }}"; +{% endif -%} diff --git a/promenade/templates/roles/common/etc/apt/sources.list.d/promenade-sources.list b/promenade/templates/roles/common/etc/apt/sources.list.d/promenade-sources.list new file mode 100644 index 00000000..e2df07d6 --- /dev/null +++ b/promenade/templates/roles/common/etc/apt/sources.list.d/promenade-sources.list @@ -0,0 +1,3 @@ +{%- for repo in config.get_path('HostSystem:packages.repositories', []) %} +{{ repo }} +{%- endfor %} diff --git a/promenade/templates/roles/common/etc/coredns/Corefile b/promenade/templates/roles/common/etc/coredns/Corefile new file mode 100644 index 00000000..cf4fdbbc --- /dev/null +++ b/promenade/templates/roles/common/etc/coredns/Corefile @@ -0,0 +1,13 @@ +promenade { + file /etc/coredns/zones/promenade + errors stdout + log stdout +} + +. { + {%- if config['KubernetesNetwork:dns.upstream_servers'] is defined %} + proxy . {%- for server in config['KubernetesNetwork:dns.upstream_servers'] %} {{ server -}}{%- endfor %} + {%- endif %} + errors stdout + log stdout +} diff --git a/promenade/templates/roles/common/etc/coredns/zones/promenade b/promenade/templates/roles/common/etc/coredns/zones/promenade new file mode 100644 index 00000000..316e9bed --- /dev/null +++ b/promenade/templates/roles/common/etc/coredns/zones/promenade @@ -0,0 +1,9 @@ +$ORIGIN promenade. + +promenade. IN SOA @ root {{ now }} 3h 15m 1w 1d + +{%- if config['KubernetesNode:join_ip'] is defined %} +apiserver.kubernetes IN A {{ config['KubernetesNode:join_ip'] }} +{%- else %} +apiserver.kubernetes IN A 127.0.0.1 +{%- endif %} diff --git a/promenade/templates/roles/common/etc/docker/daemon.json b/promenade/templates/roles/common/etc/docker/daemon.json new file mode 100644 index 00000000..4732a461 --- /dev/null +++ b/promenade/templates/roles/common/etc/docker/daemon.json @@ -0,0 +1 @@ +{{ config['Docker:config'] | tojson(indent=2) }} diff --git a/promenade/templates/roles/common/etc/hosts b/promenade/templates/roles/common/etc/hosts new file mode 100644 index 00000000..d93d1eea --- /dev/null +++ b/promenade/templates/roles/common/etc/hosts @@ -0,0 +1,7 @@ +# This file is controlled by Promenade. Do not modify. +# +127.0.0.1 {{ config.get_first('Genesis:hostname', 'KubernetesNode:hostname') }} +127.0.0.1 localhost +{%- for entry in config.get_path('KubernetesNetwork:hosts_entries', []) %} +{{ entry['ip'] }} {{ entry['names'] | join(' ') }} +{%- endfor %} diff --git a/promenade/templates/master/etc/kubernetes/admin/kubeconfig.yaml b/promenade/templates/roles/common/etc/kubernetes/admin/kubeconfig.yaml similarity index 51% rename from promenade/templates/master/etc/kubernetes/admin/kubeconfig.yaml rename to promenade/templates/roles/common/etc/kubernetes/admin/kubeconfig.yaml index f0c91bd7..87e9e634 100644 --- a/promenade/templates/master/etc/kubernetes/admin/kubeconfig.yaml +++ b/promenade/templates/roles/common/etc/kubernetes/admin/kubeconfig.yaml @@ -2,8 +2,8 @@ apiVersion: v1 clusters: - cluster: - server: https://kubernetes:6443 - certificate-authority: /etc/kubernetes/admin/pki/cluster-ca.pem + server: https://apiserver.kubernetes.promenade:6443 + certificate-authority: pki/cluster-ca.pem name: kubernetes contexts: - context: @@ -16,5 +16,5 @@ preferences: {} users: - name: admin user: - client-certificate: /etc/kubernetes/admin/pki/admin.pem - client-key: /etc/kubernetes/admin/pki/admin-key.pem + client-certificate: pki/admin.pem + client-key: pki/admin-key.pem diff --git a/promenade/templates/roles/common/etc/kubernetes/admin/pki/admin-key.pem b/promenade/templates/roles/common/etc/kubernetes/admin/pki/admin-key.pem new file mode 100644 index 00000000..d8dfc2ed --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/admin/pki/admin-key.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateKey/v1', name='admin') }} diff --git a/promenade/templates/roles/common/etc/kubernetes/admin/pki/admin.pem b/promenade/templates/roles/common/etc/kubernetes/admin/pki/admin.pem new file mode 100644 index 00000000..0c96c944 --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/admin/pki/admin.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/Certificate/v1', name='admin') }} diff --git a/promenade/templates/roles/common/etc/kubernetes/admin/pki/cluster-ca.pem b/promenade/templates/roles/common/etc/kubernetes/admin/pki/cluster-ca.pem new file mode 100644 index 00000000..0c922732 --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/admin/pki/cluster-ca.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes') }} diff --git a/promenade/templates/common/etc/kubernetes/kubelet/kubeconfig.yaml b/promenade/templates/roles/common/etc/kubernetes/kubeconfig similarity index 51% rename from promenade/templates/common/etc/kubernetes/kubelet/kubeconfig.yaml rename to promenade/templates/roles/common/etc/kubernetes/kubeconfig index 41d9e6f9..0e3dfaa8 100644 --- a/promenade/templates/common/etc/kubernetes/kubelet/kubeconfig.yaml +++ b/promenade/templates/roles/common/etc/kubernetes/kubeconfig @@ -2,8 +2,8 @@ apiVersion: v1 clusters: - cluster: - server: https://kubernetes:6443 - certificate-authority: /etc/kubernetes/kubelet/pki/cluster-ca.pem + server: https://apiserver.kubernetes.promenade:6443 + certificate-authority: pki/cluster-ca.pem name: kubernetes contexts: - context: @@ -16,5 +16,5 @@ preferences: {} users: - name: kubelet user: - client-certificate: /etc/kubernetes/kubelet/pki/kubelet.pem - client-key: /etc/kubernetes/kubelet/pki/kubelet-key.pem + client-certificate: pki/kubelet.pem + client-key: pki/kubelet-key.pem diff --git a/promenade/templates/roles/common/etc/kubernetes/manifests/coredns.yaml b/promenade/templates/roles/common/etc/kubernetes/manifests/coredns.yaml new file mode 100644 index 00000000..dfefed0a --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/manifests/coredns.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: coredns + namespace: kube-system + labels: + coredns-service: enabled + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' +spec: + hostNetwork: true + containers: + - name: coredns + image: {{ config['HostSystem:images.coredns'] }} + command: + - /coredns + - -conf + - /etc/coredns/Corefile + volumeMounts: + - name: config + mountPath: /etc/coredns + readOnly: true + volumes: + - name: config + hostPath: + path: /etc/coredns diff --git a/promenade/templates/roles/common/etc/kubernetes/pki/cluster-ca.pem b/promenade/templates/roles/common/etc/kubernetes/pki/cluster-ca.pem new file mode 100644 index 00000000..0c922732 --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/pki/cluster-ca.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes') }} diff --git a/promenade/templates/roles/common/etc/kubernetes/pki/kubelet-key.pem b/promenade/templates/roles/common/etc/kubernetes/pki/kubelet-key.pem new file mode 100644 index 00000000..35df2272 --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/pki/kubelet-key.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateKey/v1', name='kubelet-' + config.kubelet_name) }} diff --git a/promenade/templates/roles/common/etc/kubernetes/pki/kubelet.pem b/promenade/templates/roles/common/etc/kubernetes/pki/kubelet.pem new file mode 100644 index 00000000..3013e447 --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/pki/kubelet.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/Certificate/v1', name='kubelet-' + config.kubelet_name) }} diff --git a/promenade/templates/roles/common/etc/resolv.conf b/promenade/templates/roles/common/etc/resolv.conf new file mode 100644 index 00000000..0c1d2687 --- /dev/null +++ b/promenade/templates/roles/common/etc/resolv.conf @@ -0,0 +1,6 @@ +options timeout:1 attempts:1 + +nameserver 127.0.0.1 +{% for server in config['KubernetesNetwork:dns.upstream_servers'] | default([]) %} +nameserver {{ server }} +{%- endfor %} diff --git a/promenade/templates/roles/common/etc/systemd/system/docker.service.d/http-proxy.conf b/promenade/templates/roles/common/etc/systemd/system/docker.service.d/http-proxy.conf new file mode 100644 index 00000000..be5ea78a --- /dev/null +++ b/promenade/templates/roles/common/etc/systemd/system/docker.service.d/http-proxy.conf @@ -0,0 +1,4 @@ +[Service] +Environment="HTTP_PROXY={{ config['KubernetesNetwork:proxy.url'] | default('', true) }}" +Environment="HTTPS_PROXY={{ config['KubernetesNetwork:proxy.url'] | default('', true) }}" +Environment="NO_PROXY={{ config.get(kind='KubernetesNetwork') | fill_no_proxy }}" diff --git a/promenade/templates/roles/common/etc/systemd/system/kubelet.service b/promenade/templates/roles/common/etc/systemd/system/kubelet.service new file mode 100644 index 00000000..bd0ba22c --- /dev/null +++ b/promenade/templates/roles/common/etc/systemd/system/kubelet.service @@ -0,0 +1,35 @@ +[Unit] +Description=Kubernetes Worker Process +Requires=network-online.target +After=network-online.target + +[Service] +ExecStart=/opt/kubernetes/bin/kubelet \ + --allow-privileged=true \ + --cluster-domain={{ config['KubernetesNetwork:dns.cluster_domain'] }} \ + --cluster-dns={{ config['KubernetesNetwork:dns.service_ip'] }} \ + --cni-bin-dir=/opt/cni/bin \ + --cni-conf-dir=/etc/cni/net.d \ + --eviction-max-pod-grace-period -1 \ + --node-status-update-frequency 5s \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --network-plugin=cni \ + --node-ip={{ config.get_first('Genesis:ip', 'KubernetesNode:ip') }} \ + {%- if config['Genesis:labels.static'] is defined %} + --node-labels={{ config['Genesis:labels.static'] | join(',') }} \ + {%- elif config['KubernetesNode:labels.static'] is defined %} + --node-labels={{ config['KubernetesNode:labels.static'] | join(',') }} \ + {%- endif %} + --pod-manifest-path=/etc/kubernetes/manifests \ + --client-ca-file=/etc/kubernetes/pki/cluster-ca.pem \ + --tls-cert-file=/etc/kubernetes/pki/kubelet.pem \ + --tls-private-key-file=/etc/kubernetes/pki/kubelet-key.pem \ + --v=5 + +Restart=always +StartLimitInterval=0 +RestartSec=10 +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/promenade/templates/roles/common/usr/local/bin/kubectl b/promenade/templates/roles/common/usr/local/bin/kubectl new file mode 100755 index 00000000..c6404b8c --- /dev/null +++ b/promenade/templates/roles/common/usr/local/bin/kubectl @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +ADDITIONAL_DOCKER_ARGS= +TRANSLATE_CR=1 + +for item in "$@"; do + if [[ $item =~ ^-[^-].*t.* ]]; then + ADDITIONAL_DOCKER_ARGS="$ADDITIONAL_DOCKER_ARGS -t" + TRANSLATE_CR=0 + break + elif [[ $item == "--" ]]; then + break + fi +done + +if [[ $TRANSLATE_CR == 1 ]]; then + docker run --rm -i \ + $ADDITIONAL_DOCKER_ARGS \ + --net host \ + -v /etc/kubernetes/admin:/etc/kubernetes/admin \ + -e KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml \ + {{ config['HostSystem:images.kubernetes.kubectl'] }} \ + /kubectl \ + "$@" \ + | sed "s/\r//" +else + exec docker run --rm -i \ + $ADDITIONAL_DOCKER_ARGS \ + --net host \ + -v /etc/kubernetes/admin:/etc/kubernetes/admin \ + -e KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml \ + {{ config['HostSystem:images.kubernetes.kubectl'] }} \ + /kubectl \ + "$@" +fi diff --git a/promenade/templates/roles/common/usr/local/bin/promenade-teardown b/promenade/templates/roles/common/usr/local/bin/promenade-teardown new file mode 100755 index 00000000..0e7e40c3 --- /dev/null +++ b/promenade/templates/roles/common/usr/local/bin/promenade-teardown @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +{% set hostname = config.get_first('Genesis:hostname', 'KubernetesNode:hostname') -%} + +set -xe + +if [ $(kubectl get nodes | grep '\bReady\b' | wc -l) -lt 3 ]; then + echo Not enough live nodes to proceed with genesis teardown. 1>&2 + exit 1 +fi + +# Evict pods +kubectl drain --force --timeout 3600s --grace-period 1800 --ignore-daemonsets --delete-local-data {{ hostname }} || true + +kubectl label node {{ hostname }} $(kubectl get node {{ hostname }} -o 'jsonpath={.metadata.labels}' | sed 's/map\[//' | sed 's/\]//' | tr ' ' '\n' | sed 's/:.*/-/g' | grep -v 'kubernetes.io') + +sleep 30 + +# Stop kubelet +systemctl stop kubelet +systemctl disable kubelet + +# Stop remaining containers processes +docker rm -f $(docker ps -aq) || true diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/apiserver-key.pem b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/apiserver-key.pem new file mode 100644 index 00000000..92c8d64e --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/apiserver-key.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateKey/v1', name='apiserver') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/apiserver.pem b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/apiserver.pem new file mode 100644 index 00000000..b459124a --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/apiserver.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/Certificate/v1', name='apiserver') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/cluster-ca.pem b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/cluster-ca.pem new file mode 100644 index 00000000..0c922732 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/cluster-ca.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/etcd-client-ca.pem b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/etcd-client-ca.pem new file mode 100644 index 00000000..9f08e24d --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/etcd-client-ca.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes-etcd') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/etcd-client-key.pem b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/etcd-client-key.pem new file mode 100644 index 00000000..f2746c14 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/etcd-client-key.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateKey/v1', name='apiserver-etcd') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/etcd-client.pem b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/etcd-client.pem new file mode 100644 index 00000000..abe17a9c --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/etcd-client.pem @@ -0,0 +1,2 @@ +{{ config.get(schema='deckhand/Certificate/v1', name='apiserver-etcd') }} + diff --git a/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/service-account.pub b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/service-account.pub new file mode 100644 index 00000000..6f5908c0 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/apiserver/pki/service-account.pub @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/PublicKey/v1', name='service-account') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/armada/assets/manifest.yaml b/promenade/templates/roles/genesis/etc/genesis/armada/assets/manifest.yaml new file mode 100644 index 00000000..ef2df892 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/armada/assets/manifest.yaml @@ -0,0 +1 @@ +{{ config.iterate(schema='armada') | yaml_safe_dump_all | safe }} diff --git a/promenade/templates/roles/genesis/etc/genesis/armada/auth/config b/promenade/templates/roles/genesis/etc/genesis/armada/auth/config new file mode 100644 index 00000000..e4170728 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/armada/auth/config @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +clusters: +- cluster: + server: https://localhost:6443 + certificate-authority: pki/cluster-ca.pem + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: armada + name: armada@kubernetes +current-context: armada@kubernetes +kind: Config +preferences: {} +users: +- name: armada + user: + client-certificate: pki/armada.pem + client-key: pki/armada-key.pem diff --git a/promenade/templates/roles/genesis/etc/genesis/armada/auth/pki/armada-key.pem b/promenade/templates/roles/genesis/etc/genesis/armada/auth/pki/armada-key.pem new file mode 100644 index 00000000..33df233e --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/armada/auth/pki/armada-key.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateKey/v1', name='armada') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/armada/auth/pki/armada.pem b/promenade/templates/roles/genesis/etc/genesis/armada/auth/pki/armada.pem new file mode 100644 index 00000000..90c4c4e7 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/armada/auth/pki/armada.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/Certificate/v1', name='armada') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/armada/auth/pki/cluster-ca.pem b/promenade/templates/roles/genesis/etc/genesis/armada/auth/pki/cluster-ca.pem new file mode 100644 index 00000000..0c922732 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/armada/auth/pki/cluster-ca.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes') }} diff --git a/promenade/templates/master/etc/kubernetes/controller-manager/kubeconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/controller-manager/kubeconfig.yaml similarity index 50% rename from promenade/templates/master/etc/kubernetes/controller-manager/kubeconfig.yaml rename to promenade/templates/roles/genesis/etc/genesis/controller-manager/kubeconfig.yaml index 6ca9e747..a6653470 100644 --- a/promenade/templates/master/etc/kubernetes/controller-manager/kubeconfig.yaml +++ b/promenade/templates/roles/genesis/etc/genesis/controller-manager/kubeconfig.yaml @@ -2,8 +2,8 @@ apiVersion: v1 clusters: - cluster: - server: https://kubernetes:6443 - certificate-authority: /etc/kubernetes/controller-manager/pki/cluster-ca.pem + server: https://apiserver.kubernetes.promenade:6443 + certificate-authority: pki/cluster-ca.pem name: kubernetes contexts: - context: @@ -16,5 +16,5 @@ preferences: {} users: - name: controller-manager user: - client-certificate: /etc/kubernetes/controller-manager/pki/controller-manager.pem - client-key: /etc/kubernetes/controller-manager/pki/controller-manager-key.pem + client-certificate: pki/controller-manager.pem + client-key: pki/controller-manager-key.pem diff --git a/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/cluster-ca.pem b/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/cluster-ca.pem new file mode 100644 index 00000000..0c922732 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/cluster-ca.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/controller-manager-key.pem b/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/controller-manager-key.pem new file mode 100644 index 00000000..e075e272 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/controller-manager-key.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateKey/v1', name='controller-manager') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/controller-manager.pem b/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/controller-manager.pem new file mode 100644 index 00000000..c6788a28 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/controller-manager.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/Certificate/v1', name='controller-manager') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/service-account.key b/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/service-account.key new file mode 100644 index 00000000..8eca90e6 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/controller-manager/pki/service-account.key @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/PrivateKey/v1', name='service-account') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/etcd/pki/client-ca.pem b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/client-ca.pem new file mode 100644 index 00000000..9f08e24d --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/client-ca.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes-etcd') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-client-key.pem b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-client-key.pem new file mode 100644 index 00000000..bd0ee48f --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-client-key.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateKey/v1', name='kubernetes-etcd-genesis') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-client.pem b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-client.pem new file mode 100644 index 00000000..3fe3df5b --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-client.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/Certificate/v1', name='kubernetes-etcd-genesis') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-peer-key.pem b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-peer-key.pem new file mode 100644 index 00000000..97de6fd9 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-peer-key.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateKey/v1', name='kubernetes-etcd-genesis-peer') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-peer.pem b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-peer.pem new file mode 100644 index 00000000..b5d075ca --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/etcd-peer.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/Certificate/v1', name='kubernetes-etcd-genesis-peer') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/etcd/pki/peer-ca.pem b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/peer-ca.pem new file mode 100644 index 00000000..45fe1ed3 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/etcd/pki/peer-ca.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes-etcd-peer') }} diff --git a/promenade/templates/master/etc/kubernetes/scheduler/kubeconfig.yaml b/promenade/templates/roles/genesis/etc/genesis/scheduler/kubeconfig.yaml similarity index 51% rename from promenade/templates/master/etc/kubernetes/scheduler/kubeconfig.yaml rename to promenade/templates/roles/genesis/etc/genesis/scheduler/kubeconfig.yaml index 20f7dcf1..e37c6c6c 100644 --- a/promenade/templates/master/etc/kubernetes/scheduler/kubeconfig.yaml +++ b/promenade/templates/roles/genesis/etc/genesis/scheduler/kubeconfig.yaml @@ -2,8 +2,8 @@ apiVersion: v1 clusters: - cluster: - server: https://kubernetes:6443 - certificate-authority: /etc/kubernetes/scheduler/pki/cluster-ca.pem + server: https://apiserver.kubernetes.promenade:6443 + certificate-authority: pki/cluster-ca.pem name: kubernetes contexts: - context: @@ -16,5 +16,5 @@ preferences: {} users: - name: scheduler user: - client-certificate: /etc/kubernetes/scheduler/pki/scheduler.pem - client-key: /etc/kubernetes/scheduler/pki/scheduler-key.pem + client-certificate: pki/scheduler.pem + client-key: pki/scheduler-key.pem diff --git a/promenade/templates/roles/genesis/etc/genesis/scheduler/pki/cluster-ca.pem b/promenade/templates/roles/genesis/etc/genesis/scheduler/pki/cluster-ca.pem new file mode 100644 index 00000000..0c922732 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/scheduler/pki/cluster-ca.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/scheduler/pki/scheduler-key.pem b/promenade/templates/roles/genesis/etc/genesis/scheduler/pki/scheduler-key.pem new file mode 100644 index 00000000..406f5c8d --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/scheduler/pki/scheduler-key.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/CertificateKey/v1', name='scheduler') }} diff --git a/promenade/templates/roles/genesis/etc/genesis/scheduler/pki/scheduler.pem b/promenade/templates/roles/genesis/etc/genesis/scheduler/pki/scheduler.pem new file mode 100644 index 00000000..2221acca --- /dev/null +++ b/promenade/templates/roles/genesis/etc/genesis/scheduler/pki/scheduler.pem @@ -0,0 +1 @@ +{{ config.get(schema='deckhand/Certificate/v1', name='scheduler') }} diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml new file mode 100644 index 00000000..37d8e240 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: auxiliary-etcd + namespace: kube-system + labels: + component: auxiliary-etcd + promenade: genesis +spec: + hostNetwork: true +{%- with etcd_name = 'auxiliary', client_port = 12379, peer_port = 12380 %} + containers: +{% include "genesis-etcd/server-container.yaml" with context %} + - name: monitor + image: {{ config['Genesis:images.kubernetes.etcd'] }} + command: + - /bin/sh + - -c + - |- + set -x + MEMBER_COUNT=$(etcdctl member list | grep '\bstarted\b' | wc -l) + if [ $MEMBER_COUNT -gt 1 ]; then + MEMBER_ID=$(etcdctl member list | grep auxiliary | awk -F ', ' '{ print $1 }') + if [ -n $MEMBER_ID ]; then + while [ $MEMBER_COUNT -lt 4 ]; do + sleep 30 + MEMBER_COUNT=$(etcdctl member list | grep '\bstarted\b' | wc -l) + done + set -e + etcdctl member remove $MEMBER_ID + fi + + rm -rf /var/lib/etcd/* /manifests/auxiliary-kubernetes-etcd.yaml + sleep 10000 + fi + env: + - name: ETCDCTL_API + value: '3' + - name: ETCDCTL_DIAL_TIMEOUT + value: 3s + - name: ETCDCTL_ENDPOINTS + value: https://127.0.0.1:2379 + - name: ETCDCTL_CACERT + value: /etc/etcd/pki/client-ca.pem + - name: ETCDCTL_CERT + value: /etc/etcd/pki/etcd-client.pem + - name: ETCDCTL_KEY + value: /etc/etcd/pki/etcd-client-key.pem + volumeMounts: + - name: data + mountPath: /var/lib/etcd + - name: pki + mountPath: /etc/etcd/pki + - name: manifest + mountPath: /manifests + volumes: +{% include "genesis-etcd/common-volumes.yaml" with context %} + - name: manifest + hostPath: + path: /etc/kubernetes/manifests +{%- endwith %} +... diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml new file mode 100644 index 00000000..aee3d4d7 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: kubernetes-apiserver + namespace: kube-system + labels: + kubernetes-apiserver-service: enabled + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' +spec: + hostNetwork: true + containers: + - name: kube-apiserver + image: {{ config['Genesis:images.kubernetes.apiserver'] }} + command: + - /hyperkube + - apiserver + - --advertise-address={{ config['Genesis:ip'] }} + - --authorization-mode=Node,RBAC + - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds + - --anonymous-auth=false + - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem + - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem + - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem + - --kubelet-client-key=/etc/kubernetes/apiserver/pki/apiserver-key.pem + # Hard coding 3 is a pretty safe move for now. This can be exposed + # with additional configuration later. + - --apiserver-count=3 + # XXX Temporarily enabled for tiller + - --insecure-port=8080 + - --insecure-bind-address=127.0.0.1 + - --bind-address=0.0.0.0 + - --secure-port=6443 + - --runtime-config=batch/v2alpha1=true + - --allow-privileged=true + - --etcd-servers=https://localhost:2379 + - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem + - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem + - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem + - --service-cluster-ip-range={{ config['KubernetesNetwork:kubernetes.service_cidr'] }} + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub + - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem + - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem + - --v=5 + volumeMounts: + - name: config + mountPath: /etc/kubernetes/apiserver + readOnly: true + volumes: + - name: config + hostPath: + path: /etc/genesis/apiserver diff --git a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-controller-manager.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-controller-manager.yaml similarity index 76% rename from promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-controller-manager.yaml rename to promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-controller-manager.yaml index e8487a3f..4e87516d 100644 --- a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-controller-manager.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-controller-manager.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Pod metadata: - name: kube-controller-manager + name: kubernetes-controller-manager namespace: kube-system labels: tier: control-plane @@ -13,11 +13,11 @@ spec: hostNetwork: true containers: - name: kube-controller-manager - image: {{ config['Versions']['images']['kubernetes']['controller-manager'] }} + image: {{ config['Genesis:images.kubernetes.controller-manager'] }} command: - kube-controller-manager - --allocate-node-cidrs=true - - --cluster-cidr={{ config['Network']['pod_ip_cidr'] }} + - --cluster-cidr={{ config['KubernetesNetwork:kubernetes.pod_cidr'] }} - --cluster-signing-cert-file=/etc/kubernetes/controller-manager/pki/cluster-ca.pem - --cluster-signing-key-file=/etc/kubernetes/controller-manager/pki/cluster-ca-key.pem - --configure-cloud-routes=false @@ -25,7 +25,7 @@ spec: - --kubeconfig=/etc/kubernetes/controller-manager/kubeconfig.yaml - --root-ca-file=/etc/kubernetes/controller-manager/pki/cluster-ca.pem - --service-account-private-key-file=/etc/kubernetes/controller-manager/pki/service-account.key - - --service-cluster-ip-range={{ config['Network']['service_ip_cidr'] }} + - --service-cluster-ip-range={{ config['KubernetesNetwork:kubernetes.service_cidr'] }} - --use-service-account-credentials=true - --v=5 volumeMounts: @@ -35,4 +35,4 @@ spec: volumes: - name: config hostPath: - path: /etc/kubernetes/controller-manager + path: /etc/genesis/controller-manager diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-etcd.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-etcd.yaml new file mode 100644 index 00000000..8592b31b --- /dev/null +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-etcd.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: kubernetes-etcd + namespace: kube-system + labels: + # NOTE: This label needs coordination with the etcd deployed via chart. + kubernetes-etcd-service: enabled +spec: + hostNetwork: true +{%- with etcd_name = 'genesis', client_port = 2379, peer_port = 2380 %} + containers: +{% include "genesis-etcd/server-container.yaml" with context %} + volumes: +{% include "genesis-etcd/common-volumes.yaml" with context %} +{%- endwith %} +... diff --git a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-scheduler.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-scheduler.yaml similarity index 80% rename from promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-scheduler.yaml rename to promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-scheduler.yaml index 4ec8215d..2f762de5 100644 --- a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-scheduler.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-scheduler.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Pod metadata: - name: kube-scheduler + name: kubernetes-scheduler namespace: kube-system labels: tier: control-plane @@ -13,7 +13,7 @@ spec: hostNetwork: true containers: - name: kube-scheduler - image: {{ config['Versions']['images']['kubernetes']['scheduler'] }} + image: {{ config['Genesis:images.kubernetes.scheduler'] }} command: - ./hyperkube - scheduler @@ -26,4 +26,4 @@ spec: volumes: - name: config hostPath: - path: /etc/kubernetes/scheduler + path: /etc/genesis/scheduler diff --git a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/tiller.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/tiller.yaml similarity index 93% rename from promenade/templates/genesis/etc/kubernetes/kubelet/manifests/tiller.yaml rename to promenade/templates/roles/genesis/etc/kubernetes/manifests/tiller.yaml index e22d3ac1..a694e993 100644 --- a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/tiller.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/tiller.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Pod metadata: - name: tiller + name: tiller-deploy namespace: kube-system labels: app: promenade @@ -14,7 +14,7 @@ spec: - env: - name: TILLER_NAMESPACE value: kube-system - image: {{ config['Versions']['images']['tiller'] }} + image: {{ config['Genesis:images.helm.tiller'] }} command: - /tiller - -logtostderr diff --git a/promenade/templates/roles/genesis/usr/local/bin/armada b/promenade/templates/roles/genesis/usr/local/bin/armada new file mode 100755 index 00000000..60d75372 --- /dev/null +++ b/promenade/templates/roles/genesis/usr/local/bin/armada @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e + +{%- if config['KubernetesNetwork:proxy.url'] is defined %} +HTTP_PROXY={{ config['KubernetesNetwork:proxy.url'] }} +NO_PROXY={{ config.get(kind='KubernetesNetwork') | fill_no_proxy }} +{%- endif %} + + +exec docker run --rm -i \ + --net host \ + -v /etc/genesis/armada/auth:/armada/.kube \ + -v /etc/genesis/armada/assets:/etc/genesis/armada/assets \ + {%- if config.debug %} + -v /vagrant/charts:/etc/genesis/armada/assets/charts \ + {%- endif %} + {%- if config['KubernetesNetwork:proxy.url'] is defined %} + -e http_proxy=$HTTP_PROXY \ + -e HTTP_PROXY=$HTTP_PROXY \ + -e https_proxy=$HTTP_PROXY \ + -e HTTPS_PROXY=$HTTP_PROXY \ + -e no_proxy=$NO_PROXY \ + -e NO_PROXY=$NO_PROXY \ + {%- endif %} + {{ config['Genesis:images.armada'] }} \ + $* diff --git a/promenade/templates/roles/genesis/usr/local/bin/helm b/promenade/templates/roles/genesis/usr/local/bin/helm new file mode 100755 index 00000000..0f15a08c --- /dev/null +++ b/promenade/templates/roles/genesis/usr/local/bin/helm @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +if [ "x$(pwd)" = "x/" ]; then + UPDIR=/ + WORKDIR=/up +else + UPDIR=$(pwd)/.. + WORKDIR=/up/$(basename $(pwd)) +fi + +exec docker run --rm -it \ + --net host \ + -v $UPDIR:/up \ + -v /etc/kubernetes/admin:/etc/kubernetes/admin \ + -w $WORKDIR \ + -e KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml \ + {{ config['HostSystem:images.helm.helm'] }} \ + --host 127.0.0.1:44134 \ + $* diff --git a/assets/etc/kubernetes/armada-loader/assets/charts/rbac/values.yaml b/promenade/templates/roles/join/etc/kubernetes/manifests/.placeholder similarity index 100% rename from assets/etc/kubernetes/armada-loader/assets/charts/rbac/values.yaml rename to promenade/templates/roles/join/etc/kubernetes/manifests/.placeholder diff --git a/promenade/templates/roles/join/usr/local/bin/helm b/promenade/templates/roles/join/usr/local/bin/helm new file mode 100755 index 00000000..7e7b5849 --- /dev/null +++ b/promenade/templates/roles/join/usr/local/bin/helm @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e + +if [ "x$(pwd)" = "x/" ]; then + UPDIR=/ + WORKDIR=/ +else + UPDIR=$(pwd)/.. + WORKDIR=/up/$(basename $(pwd)) +fi + +exec docker run --rm -it \ + --net host \ + -v $UPDIR:/up \ + -v /etc/kubernetes/admin:/etc/kubernetes/admin \ + -w $WORKDIR \ + -e KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml \ + {{ config['HostSystem:images.helm.helm'] }} \ + $* diff --git a/promenade/templates/scripts/genesis.sh b/promenade/templates/scripts/genesis.sh new file mode 100644 index 00000000..f0cf6df7 --- /dev/null +++ b/promenade/templates/scripts/genesis.sh @@ -0,0 +1,41 @@ +{% include "header.sh" with context %} + +{% include "up.sh" with context %} + +set +x +log +log === Waiting for Kubernetes API availablity === +set -x +wait_for_kubernetes_api 3600 + +{%- if config['Genesis:labels.dynamic'] is defined %} +set +x +log +log === Registering dynamic labels for node === +set -x +register_labels {{ config['Genesis:hostname'] }} 3600 {{ config['Genesis:labels.dynamic'] | join(' ') }} +{%- endif %} + +set +x +log +log === Deploying bootstrap manifest via Armada === +set -x + +while true; do + sleep 10 + if armada apply --debug /etc/genesis/armada/assets/manifest.yaml ; then + break + fi +done + +set +x +log +log === Waiting for Node to be Ready === +set -x +wait_for_node_ready {{ config['Genesis:hostname'] }} 3600 + +{% include "cleanup.sh" with context %} + +set +x +log +log === Finished genesis process === diff --git a/promenade/templates/scripts/join.sh b/promenade/templates/scripts/join.sh new file mode 100644 index 00000000..2e0fcbfe --- /dev/null +++ b/promenade/templates/scripts/join.sh @@ -0,0 +1,25 @@ +{% include "header.sh" with context %} + +{% include "up.sh" with context %} + +set +x +log +log === Waiting for Node to be Ready === +set -x +wait_for_node_ready {{ config['KubernetesNode:hostname'] }} 3600 + +{%- if config['KubernetesNode:labels.dynamic'] is defined %} +set +x +log +log === Registering dynamic labels for node === +set -x +register_labels {{ config['KubernetesNode:hostname'] }} 3600 {{ config['KubernetesNode:labels.dynamic'] | join(' ') }} +{%- endif %} + +sleep 60 + +{% include "cleanup.sh" with context %} + +set +x +log +log === Finished join process === diff --git a/promenade/templates/scripts/validate-cluster.sh b/promenade/templates/scripts/validate-cluster.sh new file mode 100644 index 00000000..afdecfbd --- /dev/null +++ b/promenade/templates/scripts/validate-cluster.sh @@ -0,0 +1,5 @@ +{% include "header.sh" with context %} + +for node in $(kubectl get nodes -o name | cut -d / -f 2); do + validate_kubectl_logs $node +done diff --git a/promenade/templates/scripts/validate-genesis.sh b/promenade/templates/scripts/validate-genesis.sh new file mode 100644 index 00000000..c7853755 --- /dev/null +++ b/promenade/templates/scripts/validate-genesis.sh @@ -0,0 +1,3 @@ +{% include "header.sh" with context %} + +validate_kubectl_logs {{ config['Genesis:hostname'] }} diff --git a/promenade/templates/scripts/validate-join.sh b/promenade/templates/scripts/validate-join.sh new file mode 100644 index 00000000..8155f483 --- /dev/null +++ b/promenade/templates/scripts/validate-join.sh @@ -0,0 +1,3 @@ +{% include "header.sh" with context %} + +validate_kubectl_logs {{ config['KubernetesNode:hostname'] }} diff --git a/promenade/validation.py b/promenade/validation.py new file mode 100644 index 00000000..fc9eda22 --- /dev/null +++ b/promenade/validation.py @@ -0,0 +1,58 @@ +from . import exceptions +from . import logging +import jsonschema +import os +import pkg_resources +import yaml + +__all__ = ['check_schema', 'check_schemas'] + +LOG = logging.getLogger(__name__) + + +def check_schemas(documents): + for document in documents: + check_schema(document) + + +def check_schema(document): + if type(document) != dict: + LOG.error('Non-dictionary document passed to schema validation.') + return + + schema_name = document.get('schema', '') + + LOG.debug('Validating schema for schema=%s metadata.name=%s', schema_name, + document.get('metadata', {}).get('name', '')) + + if schema_name in SCHEMAS: + try: + jsonschema.validate(document.get('data'), SCHEMAS[schema_name]) + except jsonschema.ValidationError as e: + raise exceptions.ValidationException(str(e)) + else: + LOG.warning('Skipping validation for unknown schema: %s', schema_name) + + +SCHEMAS = {} + + +def _load_schemas(): + ''' + Fills the cache of known schemas + ''' + schema_dir = _get_schema_dir() + for schema_file in os.listdir(schema_dir): + with open(os.path.join(schema_dir, schema_file)) as f: + for schema in yaml.safe_load_all(f): + name = schema['metadata']['name'] + assert name not in SCHEMAS + SCHEMAS[name] = schema['data'] + + +def _get_schema_dir(): + return pkg_resources.resource_filename('promenade', 'schemas') + + +# Fill the cache +_load_schemas() diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 2265f5fb..54d3d641 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,5 +1,15 @@ +certifi==2017.7.27.1 +chardet==3.0.4 click==6.7 +decorator==4.1.2 +idna==2.6 Jinja2==2.9.6 +jsonpath-ng==1.4.3 +jsonschema==2.6.0 MarkupSafe==1.0 pbr==3.0.1 +ply==3.10 PyYAML==3.12 +requests==2.18.4 +six==1.11.0 +urllib3==1.22 diff --git a/requirements.txt b/requirements.txt index 2f841c3d..6f03da8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ click==6.7 jinja2==2.9.6 +jsonpath-ng==1.4.3 +jsonschema==2.6.0 pbr==3.0.1 pyyaml==3.12 +requests==2.18.4 diff --git a/setup.cfg b/setup.cfg index b72d247f..942f0220 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,7 @@ packages = promenade package-data = templates = templates/* + schemas = schemas/* [entry_points] console_scripts = diff --git a/test-requirements.txt b/test-requirements.txt index 38d8f59d..f90a632e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,4 @@ # Documentation +pytest sphinx>=1.6.2 sphinx_rtd_theme==0.2.4 diff --git a/tools/generated_configs/.gitignore b/tests/__init__.py similarity index 100% rename from tools/generated_configs/.gitignore rename to tests/__init__.py diff --git a/tests/test_placeholder.py b/tests/test_placeholder.py new file mode 100644 index 00000000..201975fc --- /dev/null +++ b/tests/test_placeholder.py @@ -0,0 +1,2 @@ +def test_placeholder(): + pass diff --git a/tools/build-example.sh b/tools/build-example.sh new file mode 100755 index 00000000..347d21ed --- /dev/null +++ b/tools/build-example.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -ex + +IMAGE_PROMENADE=${IMAGE_PROMENADE:-quay.io/attcomdev/promenade:latest} + +echo === Cleaning up old data === +rm -rf example/scripts +mkdir example/scripts + +echo === Generating updated certificates === +docker run --rm -t \ + -w /target \ + -e PROMENADE_DEBUG=$PROMENADE_DEBUG \ + -v $(pwd):/target \ + ${IMAGE_PROMENADE} \ + promenade \ + generate-certs \ + -o example \ + example/*.yaml + +echo === Building bootstrap scripts === +docker run --rm -t \ + -w /target \ + -e PROMENADE_DEBUG=$PROMENADE_DEBUG \ + -v $(pwd):/target \ + ${IMAGE_PROMENADE} \ + promenade \ + build-all \ + -o example/scripts \ + --validators \ + example/*.yaml + +echo === Done === diff --git a/tools/build-vagrant-box.sh b/tools/build-vagrant-box.sh deleted file mode 100755 index 82bd56dc..00000000 --- a/tools/build-vagrant-box.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -WORKDIR=$(mktemp -d) - -function cleanup { - rm -rf "${WORKDIR}" -} - -trap cleanup EXIT - -sudo apt-get update -sudo apt-get install -y --no-install-recommends \ - curl \ - unzip - -git clone https://github.com/jakobadam/packer-qemu-templates.git ${WORKDIR} - -cd ${WORKDIR}/ubuntu - -sed -i -e 's#http://releases.ubuntu.com/16.04/ubuntu-16.04-server-amd64.iso#http://old-releases.ubuntu.com/releases/xenial/ubuntu-16.04.1-server-amd64.iso' ubuntu.json - -sed -i -e 's#http://releases.ubuntu.com/16.04/ubuntu-16.04-server-amd64.iso#http -://old-releases.ubuntu.com/releases/xenial/ubuntu-16.04.1-server-amd64.iso' ubun -tu-vagrant.json - -sed -i -e 's#http://releases.ubuntu.com/16.04/ubuntu-16.04.2-server-amd64.iso#http -://old-releases.ubuntu.com/releases/xenial/ubuntu-16.04.2-server-amd64.iso' ubun -tu1604.json - -packer build -var-file=ubuntu1604.json ubuntu-vagrant.json - -vagrant box add promenade/ubuntu1604 box/libvirt/ubuntu1604-1.box diff --git a/tools/dev-build.sh b/tools/dev-build.sh index 5852cbbe..06c12287 100755 --- a/tools/dev-build.sh +++ b/tools/dev-build.sh @@ -2,22 +2,11 @@ set -ex -echo === Cleaning up old data === -rm -rf promenade.tar configs -mkdir configs +SCRIPT_DIR=$(dirname $0) echo === Building image === -docker build -t quay.io/attcomdev/promenade:latest . +docker build -t quay.io/attcomdev/promenade:latest $(realpath $SCRIPT_DIR/..) -echo === Generating updated configuration === -docker run --rm -t \ - -v $(pwd):/target quay.io/attcomdev/promenade:latest \ - promenade -v \ - generate \ - -c /target/example/vagrant-input-config.yaml \ - -o /target/configs +export PROMENADE_DEBUG=${PROMENADE_DEBUG:-1} -echo === Saving image === -docker save -o promenade.tar quay.io/attcomdev/promenade:latest - -echo === Done === +exec $SCRIPT_DIR/build-example.sh diff --git a/tools/gate/.gitignore b/tools/gate/.gitignore new file mode 100644 index 00000000..d3142aea --- /dev/null +++ b/tools/gate/.gitignore @@ -0,0 +1,4 @@ +/config +/config-env +/promenade-bundle +/promenade-bundle.tgz diff --git a/tools/gate/build.sh b/tools/gate/build.sh new file mode 100755 index 00000000..71607b39 --- /dev/null +++ b/tools/gate/build.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -ex + +GATE_DIR=$(realpath $(dirname $0)) +pushd $GATE_DIR + +ENV_PATH=$GATE_DIR/config-env +IMAGE_PROMENADE=${IMAGE_PROMENADE:-$1} + +if [ ! -s $ENV_PATH ]; then + echo Environment variables for config substitution in $ENV_PATH are required. + exit 1 +fi + +if [ "x$IMAGE_PROMENADE" = "x" ]; then + echo IMAGE_PROMENADE environment variable must be supplied. + exit 1 +fi + +echo === Building assets for testing === +echo Usinag image ${IMAGE_PROMENADE}. + +echo === Cleaning up old data === +rm -f config/* +rm -f promenade-bundle/* + +echo === Validating test environment === +env -i - $(cat default-config-env) env $(cat $ENV_PATH) $GATE_DIR/util/validate-test-env.sh + +echo === Substituting variables into configuration === +for template in config-templates/*; do + OUTPUT_PATH=config/$(basename $template) + env -i - $(cat default-config-env) env IMAGE_PROMENADE=$IMAGE_PROMENADE $(cat $ENV_PATH) envsubst < $template > $OUTPUT_PATH + + cat $OUTPUT_PATH + echo + echo +done + +echo === Generating certificates === +docker run --rm -t \ + -w /target \ + -v $GATE_DIR:/target \ + ${IMAGE_PROMENADE} \ + promenade \ + generate-certs \ + -o config \ + config/*.yaml + +echo === Building genesis and join scripts +docker run --rm -t \ + -w /target \ + -v $GATE_DIR:/target \ + ${IMAGE_PROMENADE} \ + promenade \ + build-all \ + --validators \ + -o promenade-bundle \ + config/*.yaml + +echo === Bundling assets for delivery === +cp $GATE_DIR/final-validation.sh promenade-bundle +tar czf promenade-bundle.tgz promenade-bundle/* + +echo === Done === diff --git a/tools/gate/config-templates/bootstrap-armada-config.yaml b/tools/gate/config-templates/bootstrap-armada-config.yaml new file mode 100644 index 00000000..eac9511e --- /dev/null +++ b/tools/gate/config-templates/bootstrap-armada-config.yaml @@ -0,0 +1,958 @@ +--- +schema: armada/Manifest/v1 +metadata: + schema: metadata/Document/v1 + name: cluster-bootstrap + layeringDefinition: + abstract: false + layer: site +data: + release_prefix: ucp + chart_groups: + - kubernetes-proxy + - container-networking + - dns + - kubernetes + - kubernetes-rbac +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-proxy + layeringDefinition: + abstract: false + layer: site +data: + description: Kubernetes proxy + sequenced: true + chart_group: + - kubernetes-proxy +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: container-networking + layeringDefinition: + abstract: false + layer: site +data: + description: Container networking via Calico + sequenced: true + chart_group: + - calico-etcd + - calico +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: dns + layeringDefinition: + abstract: false + layer: site +data: + description: Cluster DNS + chart_group: + - coredns +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-rbac + layeringDefinition: + abstract: false + layer: site +data: + description: Role Based Access Control configuration for Kubernetes + sequenced: true + chart_group: + - kubernetes-rbac +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes + layeringDefinition: + abstract: false + layer: site +data: + description: Kubernetes components + chart_group: + - kubernetes-etcd + - kubernetes-apiserver + - kubernetes-controller-manager + - kubernetes-scheduler +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: helm-toolkit +data: + chart_name: helm-toolkit + release: helm-toolkit + namespace: helm-toolkit + timeout: 600 + upgrade: + no_hooks: true + values: {} + source: + type: git + location: https://git.openstack.org/openstack/openstack-helm + subpath: helm-toolkit + reference: master + dependencies: [] +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-proxy + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: $ + dest: + path: '$.values.tls.ca' + - + src: + schema: deckhand/Certificate/v1 + name: proxy + path: $ + dest: + path: '$.values.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: proxy + path: $ + dest: + path: '$.values.tls.key' +data: + chart_name: proxy + release: kubernetes-proxy + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + tls: + ca: placeholder + cert: placeholder + key: placeholder + images: + proxy: ${IMAGE_HYPERKUBE} + network: + kubernetes_netloc: apiserver.kubernetes.promenade:6443 + pod_cidr: 10.97.0.0/16 + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: proxy + dependencies: [] +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: calico-etcd + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: calico-etcd + path: $ + dest: + path: '$.values.tls.client.ca' + - + src: + schema: deckhand/CertificateAuthority/v1 + name: calico-etcd-peer + path: $ + dest: + path: '$.values.tls.peer.ca' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-anchor + path: $ + dest: + path: '$.values.anchor.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-anchor + path: $ + dest: + path: '$.values.anchor.tls.key' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-${GENESIS_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[0].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-${GENESIS_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[0].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-${GENESIS_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[0].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-${GENESIS_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[0].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-${MASTER1_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[1].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-${MASTER1_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[1].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-${MASTER1_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[1].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-${MASTER1_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[1].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-${MASTER2_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[2].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-${MASTER2_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[2].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-${MASTER2_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[2].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-${MASTER2_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[2].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-${WORKER_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[3].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-${WORKER_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[3].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: calico-etcd-${WORKER_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[3].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-etcd-${WORKER_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[3].tls.peer.key' + +data: + chart_name: etcd + release: calico-etcd + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + anchor: + etcdctl_endpoint: 10.96.232.136 + node_selector: + key: calico-etcd + value: enabled + tls: + cert: placeholder + key: placeholder + etcd: + host_data_path: /var/lib/etcd/calico + host_etc_path: /etc/etcd/calico + bootstrapping: + enabled: true + host_directory: /var/lib/anchor + filename: calico-etcd-bootstrap + images: + etcd: ${IMAGE_ETCD} + etcdctl: ${IMAGE_ETCD} + nodes: + - name: ${GENESIS_HOSTNAME} + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + - name: ${MASTER1_HOSTNAME} + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + - name: ${MASTER2_HOSTNAME} + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + # ${WORKER_HOSTNAME} is here to demonstrate movability of the cluster + - name: ${WORKER_HOSTNAME} + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + service: + name: calico-etcd + ip: 10.96.232.136 + client: + port: 6666 + target_port: 6666 + peer: + port: 6667 + target_port: 6667 + tls: + client: + ca: placeholder + peer: + ca: placeholder + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: etcd + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: calico + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: calico-etcd + path: $ + dest: + path: '$.values.etcd.tls.ca' + - + src: + schema: deckhand/Certificate/v1 + name: calico-node + path: $ + dest: + path: '$.values.etcd.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: calico-node + path: $ + dest: + path: '$.values.etcd.tls.key' +data: + chart_name: calico + release: calico + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + calico: + ip_autodetection_method: ${CALICO_IP_AUTODETECTION_METHOD} + pod_ip_cidr: 10.97.0.0/16 + ctl: + install_on_host: true + etcd: + service: + ip: 10.96.232.136 + port: 6666 + tls: + ca: placeholder + cert: placeholder + key: placeholder + images: + cni: ${IMAGE_CALICO_CNI} + ctl: ${IMAGE_CALICO_CTL} + node: ${IMAGE_CALICO_NODE} + policy_controller: ${IMAGE_CALICO_KUBE_CONTROLLERS} + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: calico + dependencies: [] +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: coredns + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: $ + dest: + path: '$.values.tls.ca' + - + src: + schema: deckhand/Certificate/v1 + name: coredns + path: $ + dest: + path: '$.values.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: coredns + path: $ + dest: + path: '$.values.tls.key' +data: + chart_name: coredns + release: coredns + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + coredns: + cluster_domain: cluster.local + upstream_nameservers: + - 8.8.8.8 + - 8.8.4.4 + + zones: + - name: promenade + services: + - bind_name: apiserver.kubernetes + service: + name: kubernetes-apiserver + namespace: kube-system + - bind_name: etcd.kubernetes + service: + name: kubernetes-etcd + namespace: kube-system + - bind_name: etcd.calico + service: + name: calico-etcd + namespace: kube-system + images: + anchor: ${IMAGE_HYPERKUBE} + coredns: ${IMAGE_COREDNS} + tls: + ca: placeholder + cert: placeholder + key: placeholder + network: + kubernetes_netloc: apiserver.kubernetes.promenade:6443 + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: coredns + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-apiserver + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: $ + dest: + path: $.values.tls.ca + + - + src: + schema: deckhand/Certificate/v1 + name: apiserver + path: $ + dest: + path: $.values.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: apiserver + path: $ + dest: + path: $.values.tls.key + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes-etcd + path: $ + dest: + path: $.values.apiserver.etcd.tls.ca + - + src: + schema: deckhand/Certificate/v1 + name: apiserver-etcd + path: $ + dest: + path: $.values.apiserver.etcd.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: apiserver-etcd + path: $ + dest: + path: $.values.apiserver.etcd.tls.key + - + src: + schema: deckhand/PublicKey/v1 + name: service-account + path: $ + dest: + path: $.values.service_account.public_key + +data: + chart_name: apiserver + release: kubernetes-apiserver + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + apiserver: + etcd: + endpoints: https://etcd.kubernetes.promenade:2379 + tls: + ca: placeholder + cert: placeholder + key: placeholder + images: + anchor: ${IMAGE_HYPERKUBE} + apiserver: ${IMAGE_HYPERKUBE} + service_account: + public_key: placeholder + tls: + ca: placeholder + cert: placeholder + key: placeholder + network: + kubernetes_service_ip: 10.96.0.1 + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: apiserver + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-controller-manager + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: $ + dest: + path: $.values.tls.ca + + - + src: + schema: deckhand/Certificate/v1 + name: controller-manager + path: $ + dest: + path: $.values.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: controller-manager + path: $ + dest: + path: $.values.tls.key + - + src: + schema: deckhand/PrivateKey/v1 + name: service-account + path: $ + dest: + path: $.values.service_account.private_key + +data: + chart_name: controller_manager + release: kubernetes-controller-manager + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + images: + anchor: ${IMAGE_HYPERKUBE} + controller_manager: ${IMAGE_HYPERKUBE} + service_account: + private_key: placeholder + tls: + ca: placeholder + cert: placeholder + key: placeholder + network: + kubernetes_netloc: apiserver.kubernetes.promenade:6443 + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: controller_manager + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-scheduler + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes + path: $ + dest: + path: $.values.tls.ca + - + src: + schema: deckhand/Certificate/v1 + name: scheduler + path: $ + dest: + path: $.values.tls.cert + - + src: + schema: deckhand/CertificateKey/v1 + name: scheduler + path: $ + dest: + path: $.values.tls.key + +data: + chart_name: scheduler + release: kubernetes-scheduler + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + tls: + ca: placeholder + cert: placeholder + key: placeholder + + network: + kubernetes_netloc: apiserver.kubernetes.promenade:6443 + + images: + anchor: ${IMAGE_HYPERKUBE} + scheduler: ${IMAGE_HYPERKUBE} + + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: scheduler + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-etcd + layeringDefinition: + abstract: false + layer: site + substitutions: + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes-etcd + path: $ + dest: + path: '$.values.tls.client.ca' + - + src: + schema: deckhand/CertificateAuthority/v1 + name: kubernetes-etcd-peer + path: $ + dest: + path: '$.values.tls.peer.ca' + + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-anchor + path: $ + dest: + path: '$.values.anchor.tls.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-anchor + path: $ + dest: + path: '$.values.anchor.tls.key' + + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-${GENESIS_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[0].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-${GENESIS_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[0].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-${GENESIS_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[0].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-${GENESIS_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[0].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-${MASTER1_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[1].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-${MASTER1_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[1].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-${MASTER1_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[1].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-${MASTER1_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[1].tls.peer.key' + + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-${MASTER2_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[2].tls.client.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-${MASTER2_HOSTNAME} + path: $ + dest: + path: '$.values.nodes[2].tls.client.key' + - + src: + schema: deckhand/Certificate/v1 + name: kubernetes-etcd-${MASTER2_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[2].tls.peer.cert' + - + src: + schema: deckhand/CertificateKey/v1 + name: kubernetes-etcd-${MASTER2_HOSTNAME}-peer + path: $ + dest: + path: '$.values.nodes[2].tls.peer.key' + +data: + chart_name: etcd + release: kubernetes-etcd + namespace: kube-system + timeout: 600 + upgrade: + no_hooks: true + values: + anchor: + etcdctl_endpoint: 10.96.0.2 + node_selector: + key: kubernetes-etcd + value: enabled + tls: + cert: placeholder + key: placeholder + etcd: + host_data_path: /var/lib/etcd/kubernetes + host_etc_path: /etc/etcd/kubernetes + images: + etcd: ${IMAGE_ETCD} + etcdctl: ${IMAGE_ETCD} + nodes: + - name: ${GENESIS_HOSTNAME} + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + - name: ${MASTER1_HOSTNAME} + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + - name: ${MASTER2_HOSTNAME} + tls: + client: + cert: placeholder + key: placeholder + peer: + cert: placeholder + key: placeholder + service: + name: kubernetes-etcd + ip: 10.96.0.2 + client: + port: 2379 + target_port: 2379 + peer: + port: 2380 + target_port: 2380 + tls: + client: + ca: placeholder + peer: + ca: placeholder + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: etcd + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-rbac + layeringDefinition: + abstract: false + layer: site +data: + chart_name: rbac + release: rbac + namespace: kube-system + timeout: 600 + values: {} + upgrade: + no_hooks: true + source: + type: local + location: /etc/genesis/armada/assets/charts + subpath: rbac + dependencies: [] +... diff --git a/tools/gate/config-templates/genesis-config.yaml b/tools/gate/config-templates/genesis-config.yaml new file mode 100644 index 00000000..3908bca7 --- /dev/null +++ b/tools/gate/config-templates/genesis-config.yaml @@ -0,0 +1,36 @@ +--- +schema: promenade/Genesis/v1 +metadata: + schema: metadata/Document/v1 + name: genesis + layeringDefinition: + abstract: false + layer: site +data: + hostname: ${GENESIS_HOSTNAME} + ip: ${GENESIS_IP} + labels: + static: + - calico-etcd=enabled + - node-role.kubernetes.io/master= + dynamic: + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - promenade-genesis=enabled + - ucp-control-plane=enabled + images: + armada: ${IMAGE_ARMADA} + helm: + tiller: ${IMAGE_TILLER} + kubernetes: + apiserver: ${IMAGE_HYPERKUBE} + controller-manager: ${IMAGE_HYPERKUBE} + etcd: ${IMAGE_ETCD} + scheduler: ${IMAGE_HYPERKUBE} + files: + - path: /var/lib/anchor/calico-etcd-bootstrap + content: "" + mode: 0644 +... diff --git a/tools/gate/config-templates/joining-host-config.yaml b/tools/gate/config-templates/joining-host-config.yaml new file mode 100644 index 00000000..6f7673ab --- /dev/null +++ b/tools/gate/config-templates/joining-host-config.yaml @@ -0,0 +1,82 @@ +--- +schema: promenade/KubernetesNode/v1 +metadata: + schema: metadata/Document/v1 + name: ${GENESIS_HOSTNAME} + layeringDefinition: + abstract: false + layer: site +data: + hostname: ${GENESIS_HOSTNAME} + ip: ${GENESIS_IP} + join_ip: ${MASTER1_IP} + labels: + static: + - node-role.kubernetes.io/master= + dynamic: + - calico-etcd=enabled + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - ucp-control-plane=enabled +--- +schema: promenade/KubernetesNode/v1 +metadata: + schema: metadata/Document/v1 + name: ${MASTER1_HOSTNAME} + layeringDefinition: + abstract: false + layer: site +data: + hostname: ${MASTER1_HOSTNAME} + ip: ${MASTER1_IP} + join_ip: ${GENESIS_IP} + labels: + static: + - node-role.kubernetes.io/master= + dynamic: + - calico-etcd=enabled + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - ucp-control-plane=enabled +--- +schema: promenade/KubernetesNode/v1 +metadata: + schema: metadata/Document/v1 + name: ${MASTER2_HOSTNAME} + layeringDefinition: + abstract: false + layer: site +data: + hostname: ${MASTER2_HOSTNAME} + ip: ${MASTER2_IP} + join_ip: ${GENESIS_IP} + labels: + static: + - node-role.kubernetes.io/master= + dynamic: + - calico-etcd=enabled + - kubernetes-apiserver=enabled + - kubernetes-controller-manager=enabled + - kubernetes-etcd=enabled + - kubernetes-scheduler=enabled + - ucp-control-plane=enabled +--- +schema: promenade/KubernetesNode/v1 +metadata: + schema: metadata/Document/v1 + name: ${WORKER_HOSTNAME} + layeringDefinition: + abstract: false + layer: site +data: + hostname: ${WORKER_HOSTNAME} + ip: ${WORKER_IP} + join_ip: ${MASTER1_IP} + labels: + dynamic: + - ucp-control-plane=enabled +... diff --git a/tools/gate/config-templates/site-config.yaml b/tools/gate/config-templates/site-config.yaml new file mode 100644 index 00000000..938ddba4 --- /dev/null +++ b/tools/gate/config-templates/site-config.yaml @@ -0,0 +1,100 @@ +--- +schema: promenade/KubernetesNetwork/v1 +metadata: + schema: metadata/Document/v1 + name: kubernetes-network + layeringDefinition: + abstract: false + layer: site +data: + dns: + cluster_domain: cluster.local + service_ip: 10.96.0.10 + bootstrap_validation_checks: + - calico-etcd.kube-system.svc.cluster.local + - kubernetes-etcd.kube-system.svc.cluster.local + - kubernetes.default.svc.cluster.local + upstream_servers: + - 8.8.8.8 + - 8.8.4.4 + + kubernetes: + pod_cidr: 10.97.0.0/16 + service_cidr: 10.96.0.0/16 + service_ip: 10.96.0.1 + + etcd: + service_ip: 10.96.0.2 +--- +schema: promenade/Docker/v1 +metadata: + schema: metadata/Document/v1 + name: docker + layeringDefinition: + abstract: false + layer: site +data: + config: + insecure-registries: + - 10.24.20.19:30092 + live-restore: true + storage-driver: overlay2 +--- +schema: promenade/HostSystem/v1 +metadata: + schema: metadata/Document/v1 + name: host-system + layeringDefinition: + abstract: false + layer: site +data: + files: + - path: /opt/kubernetes/bin/kubelet + tar_url: ${KUBELET_URL} + tar_path: kubernetes/node/bin/kubelet + mode: 0555 + images: + coredns: ${IMAGE_COREDNS} + helm: + helm: ${IMAGE_HELM} + kubernetes: + kubectl: ${IMAGE_HYPERKUBE} + packages: + repositories: + - deb http://apt.dockerproject.org/repo ubuntu-xenial main + keys: + - |- + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFWln24BEADrBl5p99uKh8+rpvqJ48u4eTtjeXAWbslJotmC/CakbNSqOb9o + ddfzRvGVeJVERt/Q/mlvEqgnyTQy+e6oEYN2Y2kqXceUhXagThnqCoxcEJ3+KM4R + mYdoe/BJ/J/6rHOjq7Omk24z2qB3RU1uAv57iY5VGw5p45uZB4C4pNNsBJXoCvPn + TGAs/7IrekFZDDgVraPx/hdiwopQ8NltSfZCyu/jPpWFK28TR8yfVlzYFwibj5WK + dHM7ZTqlA1tHIG+agyPf3Rae0jPMsHR6q+arXVwMccyOi+ULU0z8mHUJ3iEMIrpT + X+80KaN/ZjibfsBOCjcfiJSB/acn4nxQQgNZigna32velafhQivsNREFeJpzENiG + HOoyC6qVeOgKrRiKxzymj0FIMLru/iFF5pSWcBQB7PYlt8J0G80lAcPr6VCiN+4c + NKv03SdvA69dCOj79PuO9IIvQsJXsSq96HB+TeEmmL+xSdpGtGdCJHHM1fDeCqkZ + hT+RtBGQL2SEdWjxbF43oQopocT8cHvyX6Zaltn0svoGs+wX3Z/H6/8P5anog43U + 65c0A+64Jj00rNDr8j31izhtQMRo892kGeQAaaxg4Pz6HnS7hRC+cOMHUU4HA7iM + zHrouAdYeTZeZEQOA7SxtCME9ZnGwe2grxPXh/U/80WJGkzLFNcTKdv+rwARAQAB + tDdEb2NrZXIgUmVsZWFzZSBUb29sIChyZWxlYXNlZG9ja2VyKSA8ZG9ja2VyQGRv + Y2tlci5jb20+iQI4BBMBAgAiBQJVpZ9uAhsvBgsJCAcDAgYVCAIJCgsEFgIDAQIe + AQIXgAAKCRD3YiFXLFJgnbRfEAC9Uai7Rv20QIDlDogRzd+Vebg4ahyoUdj0CH+n + Ak40RIoq6G26u1e+sdgjpCa8jF6vrx+smpgd1HeJdmpahUX0XN3X9f9qU9oj9A4I + 1WDalRWJh+tP5WNv2ySy6AwcP9QnjuBMRTnTK27pk1sEMg9oJHK5p+ts8hlSC4Sl + uyMKH5NMVy9c+A9yqq9NF6M6d6/ehKfBFFLG9BX+XLBATvf1ZemGVHQusCQebTGv + 0C0V9yqtdPdRWVIEhHxyNHATaVYOafTj/EF0lDxLl6zDT6trRV5n9F1VCEh4Aal8 + L5MxVPcIZVO7NHT2EkQgn8CvWjV3oKl2GopZF8V4XdJRl90U/WDv/6cmfI08GkzD + YBHhS8ULWRFwGKobsSTyIvnbk4NtKdnTGyTJCQ8+6i52s+C54PiNgfj2ieNn6oOR + 7d+bNCcG1CdOYY+ZXVOcsjl73UYvtJrO0Rl/NpYERkZ5d/tzw4jZ6FCXgggA/Zxc + jk6Y1ZvIm8Mt8wLRFH9Nww+FVsCtaCXJLP8DlJLASMD9rl5QS9Ku3u7ZNrr5HWXP + HXITX660jglyshch6CWeiUATqjIAzkEQom/kEnOrvJAtkypRJ59vYQOedZ1sFVEL + MXg2UCkD/FwojfnVtjzYaTCeGwFQeqzHmM241iuOmBYPeyTY5veF49aBJA1gEJOQ + TvBR8Q== + =Fm3p + -----END PGP PUBLIC KEY BLOCK----- + additional: [] + required: + docker: docker-engine=1.13.1-0~ubuntu-xenial + socat: socat=1.7.3.1-1 +... diff --git a/tools/gate/config/.gitkeep b/tools/gate/config/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tools/gate/default-config-env b/tools/gate/default-config-env new file mode 100644 index 00000000..b868f596 --- /dev/null +++ b/tools/gate/default-config-env @@ -0,0 +1,11 @@ +IMAGE_ARMADA=quay.io/attcomdev/armada:latest +IMAGE_CALICO_CNI=quay.io/calico/cni:v1.11.0 +IMAGE_CALICO_CTL=quay.io/calico/ctl:v1.6.1 +IMAGE_CALICO_KUBE_CONTROLLERS=quay.io/calico/kube-controllers:v1.0.0 +IMAGE_CALICO_NODE=quay.io/calico/node:v2.6.1 +IMAGE_COREDNS=coredns/coredns:011 +IMAGE_ETCD=quay.io/coreos/etcd:v3.0.17 +IMAGE_HELM=lachlanevenson/k8s-helm:v2.5.1 +IMAGE_HYPERKUBE=gcr.io/google_containers/hyperkube-amd64:v1.8.0 +IMAGE_TILLER=gcr.io/kubernetes-helm/tiller:v2.5.1 +KUBELET_URL=https://dl.k8s.io/v1.8.0/kubernetes-node-linux-amd64.tar.gz diff --git a/tools/gate/final-validation.sh b/tools/gate/final-validation.sh new file mode 100755 index 00000000..b099610a --- /dev/null +++ b/tools/gate/final-validation.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +set -ex diff --git a/tools/gate/promenade-bundle/.gitkeep b/tools/gate/promenade-bundle/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tools/gate/required-config-env b/tools/gate/required-config-env new file mode 100644 index 00000000..117781a4 --- /dev/null +++ b/tools/gate/required-config-env @@ -0,0 +1,9 @@ +CALICO_IP_AUTODETECTION_METHOD +GENESIS_HOSTNAME +GENESIS_IP +MASTER1_HOSTNAME +MASTER1_IP +MASTER2_HOSTNAME +MASTER2_IP +WORKER_HOSTNAME +WORKER_IP diff --git a/tools/gate/util/validate-test-env.sh b/tools/gate/util/validate-test-env.sh new file mode 100755 index 00000000..bc2b7c68 --- /dev/null +++ b/tools/gate/util/validate-test-env.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR=$(dirname $0) +REQUIRED_VARIABLES_PATH=$(realpath $SCRIPT_DIR/../required-config-env) + +env | sort + +for varname in $(cat $REQUIRED_VARIABLES_PATH); do + if [ "x${!varname}" = "x" ]; then + echo Missing required variable: $varname + exit 1 + fi +done diff --git a/tools/promenade-config.yaml b/tools/promenade-config.yaml deleted file mode 100644 index 787bb14c..00000000 --- a/tools/promenade-config.yaml +++ /dev/null @@ -1,90 +0,0 @@ ---- -apiVersion: promenade/v1 -kind: Cluster -metadata: - name: example - target: none -spec: - nodes: - G_HOSTNAME: - ip: GENESIS - kubernetes_interface: G_IFACE - roles: - - master - - genesis - additional_labels: - - beta.kubernetes.io/arch=amd64 - M1_HOSTNAME: - ip: MASTER_1 - kubernetes_interface: M1_IFACE - roles: - - master - additional_labels: - - beta.kubernetes.io/arch=amd64 - M2_HOSTNAME: - ip: MASTER_2 - kubernetes_interface: M2_IFACE - roles: - - master - additional_labels: - - beta.kubernetes.io/arch=amd64 - W_HOSTNAME: - ip: WORKER - kubernetes_interface: W_IFACE - roles: - - worker - additional_labels: - - beta.kubernetes.io/arch=amd64 ---- -apiVersion: promenade/v1 -kind: Network -metadata: - cluster: example - name: example - target: all -spec: - cluster_domain: cluster.local - cluster_dns: 10.96.0.10 - kube_service_ip: 10.96.0.1 - pod_ip_cidr: 10.97.0.0/16 - service_ip_cidr: 10.96.0.0/16 - calico_etcd_service_ip: 10.96.232.136 - dns_servers: - - 8.8.8.8 - - 8.8.4.4 - #http_proxy: http://proxy.example.com:8080 - #https_proxy: https://proxy.example.com:8080 ---- -apiVersion: promenade/v1 -kind: Versions -metadata: - cluster: example - name: example - target: all -spec: - images: - armada: quay.io/attcomdev/armada:latest - calico: - cni: quay.io/calico/cni:v1.9.1 - etcd: quay.io/coreos/etcd:v3.2.1 - node: quay.io/calico/node:v1.3.0 - policy-controller: quay.io/calico/kube-policy-controller:v0.6.0 - kubernetes: - apiserver: gcr.io/google_containers/hyperkube-amd64:v1.6.8 - controller-manager: quay.io/attcomdev/kube-controller-manager:v1.6.8 - dns: - dnsmasq: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.2 - kubedns: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.2 - sidecar: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.2 - etcd: quay.io/coreos/etcd:v3.2.1 - kubectl: gcr.io/google_containers/hyperkube-amd64:v1.6.8 - proxy: gcr.io/google_containers/hyperkube-amd64:v1.6.8 - scheduler: gcr.io/google_containers/hyperkube-amd64:v1.6.8 - promenade: quay.io/attcomdev/promenade:latest - tiller: gcr.io/kubernetes-helm/tiller:v2.5.0 - packages: - docker: docker.io=1.12.6-0ubuntu1~16.04.1 - dnsmasq: dnsmasq=2.75-1ubuntu0.16.04.2 - socat: socat=1.7.3.1-1 - additional_packages: - - ceph-common=10.2.7-0ubuntu0.16.04.1 diff --git a/tools/registry/IMAGES b/tools/registry/IMAGES new file mode 100644 index 00000000..65efd537 --- /dev/null +++ b/tools/registry/IMAGES @@ -0,0 +1,14 @@ +# source_name, tag, cache_name +coredns/coredns,011,coredns +gcr.io/google_containers/hyperkube-amd64,v1.8.0,hyperkube +gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64,1.14.4,k8s-dns-dnsmasq-nanny-amd64 +gcr.io/google_containers/k8s-dns-kube-dns-amd64,1.14.4,k8s-dns-kube-dns-amd64 +gcr.io/google_containers/k8s-dns-sidecar-amd64,1.14.4,k8s-dns-sidecar-amd64 +gcr.io/kubernetes-helm/tiller,v2.5.1,tiller +lachlanevenson/k8s-helm,v2.5.1,helm +quay.io/attcomdev/armada,latest,armada +quay.io/calico/cni,v1.11.0,calico-cni +quay.io/calico/ctl,v1.6.1,calico-ctl +quay.io/calico/kube-controllers,v1.0.0,calico-kube-controllers +quay.io/calico/node,v2.6.1,calico-node +quay.io/coreos/etcd,v3.0.17,etcd diff --git a/tools/registry/revert_example.sh b/tools/registry/revert_example.sh new file mode 100755 index 00000000..fc94cb26 --- /dev/null +++ b/tools/registry/revert_example.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -ex + +IMAGES_FILE=$(dirname $0)/IMAGES + +IFS=, +grep -v '^#.*' $IMAGES_FILE | while read src tag dst; do + sed -i "s;registry:5000/$dst:$tag;$src:$tag;g" example/*.yaml +done diff --git a/tools/registry/start.sh b/tools/registry/start.sh new file mode 100755 index 00000000..93314dd6 --- /dev/null +++ b/tools/registry/start.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -ex + +REGISTRY_DATA_DIR=${REGISTRY_DATA_DIR:-/mnt/registry} + +docker run -d \ + -p 5000:5000 \ + -e REGISTRY_HTTP_ADDR=0.0.0.0:5000 \ + --restart=always \ + --name registry \ + -v $REGISTRY_DATA_DIR:/var/lib/registry \ + registry:2 diff --git a/tools/registry/stop.sh b/tools/registry/stop.sh new file mode 100755 index 00000000..ef2cced5 --- /dev/null +++ b/tools/registry/stop.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -ex + +docker rm -fv registry diff --git a/tools/registry/update_cache.sh b/tools/registry/update_cache.sh new file mode 100755 index 00000000..3e289d56 --- /dev/null +++ b/tools/registry/update_cache.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -ex + +IMAGES_FILE=$(dirname $0)/IMAGES + +IFS=, +grep -v '^#.*' $IMAGES_FILE | while read src tag dst; do + echo src=$src tag=$tag dst=$dst + sudo docker pull $src:$tag + + full_dst=localhost:5000/$dst:$tag + sudo docker tag $src:$tag $full_dst + + sudo docker push $full_dst +done diff --git a/tools/registry/update_example.sh b/tools/registry/update_example.sh new file mode 100755 index 00000000..21621dd1 --- /dev/null +++ b/tools/registry/update_example.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -ex + +IMAGES_FILE=$(dirname $0)/IMAGES + +IFS=, +grep -v '^#.*' $IMAGES_FILE | while read src tag dst; do + sed -i "s;$src:$tag;registry:5000/$dst:$tag;g" example/*.yaml +done diff --git a/tools/vagrant/build-vagrant-box.sh b/tools/vagrant/build-vagrant-box.sh new file mode 100755 index 00000000..85f4f69b --- /dev/null +++ b/tools/vagrant/build-vagrant-box.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -ex + +WORKDIR=$(mktemp -d) + +function cleanup { + rm -rf "${WORKDIR}" +} + +trap cleanup EXIT + +sudo apt-get update +sudo apt-get install -y --no-install-recommends \ + curl \ + unzip + +git clone https://github.com/jakobadam/packer-qemu-templates.git ${WORKDIR} + +cd ${WORKDIR}/ubuntu + + +sed -i -e 's#http://releases.ubuntu.com/16.04/ubuntu-16.04-server-amd64.iso#http://old-releases.ubuntu.com/releases/xenial/ubuntu-16.04.2-server-amd64.iso#g' ubuntu.json +sed -i -e 's/de5ee8665048f009577763efbf4a6f0558833e59/f529548fa7468f2d8413b8427d8e383b830df5f6/g' ubuntu.json +sed -i -e 's#http://releases.ubuntu.com/16.04/ubuntu-16.04.1-server-amd64.iso#http://old-releases.ubuntu.com/releases/xenial/ubuntu-16.04.2-server-amd64.iso#g' ubuntu-vagrant.json +sed -i -e 's/de5ee8665048f009577763efbf4a6f0558833e59/f529548fa7468f2d8413b8427d8e383b830df5f6/g' ubuntu-vagrant.json +sed -i -e 's#http://releases.ubuntu.com/16.04/ubuntu-16.04.3-server-amd64.iso#http://old-releases.ubuntu.com/releases/xenial/ubuntu-16.04.2-server-amd64.iso#g' ubuntu1604.json +sed -i -e 's/a06cd926f5855d4f21fb4bc9978a35312f815fbda0d0ef7fdc846861f4fc4600/737ae7041212c628de5751d15c3016058b0e833fdc32e7420209b76ca3d0a535/g' ubuntu1604.json +sed -i -e 's#http://releases.ubuntu.com/16.04/ubuntu-16.04-server-amd64.iso#http://old-releases.ubuntu.com/releases/xenial/ubuntu-16.04.1-server-amd64.iso#g' ubuntu.json + +PACKER_LOG="yes" packer build -var-file=ubuntu1604.json ubuntu-vagrant.json + +vagrant box add promenade/ubuntu1604 box/libvirt/ubuntu1604-1.box diff --git a/tools/full-vagrant-setup.sh b/tools/vagrant/full-vagrant-setup.sh similarity index 100% rename from tools/full-vagrant-setup.sh rename to tools/vagrant/full-vagrant-setup.sh diff --git a/tools/install-packer.sh b/tools/vagrant/install-packer.sh similarity index 100% rename from tools/install-packer.sh rename to tools/vagrant/install-packer.sh diff --git a/tools/install-vagrant-libvirt.sh b/tools/vagrant/install-vagrant-libvirt.sh similarity index 100% rename from tools/install-vagrant-libvirt.sh rename to tools/vagrant/install-vagrant-libvirt.sh diff --git a/tools/install-vagrant-nfs-deps.sh b/tools/vagrant/install-vagrant-nfs-deps.sh similarity index 100% rename from tools/install-vagrant-nfs-deps.sh rename to tools/vagrant/install-vagrant-nfs-deps.sh diff --git a/tox.ini b/tox.ini index f1e5b5a2..2a9a6c91 100644 --- a/tox.ini +++ b/tox.ini @@ -15,3 +15,9 @@ whitelist_externals=rm commands = rm -rf docs/build python setup.py build_sphinx {posargs} + +[testenv:freeze] +deps = -r{toxinidir}/requirements.txt +whitelist_externals=sh +commands= + sh -c "pip freeze | grep -v '^promenade' > {toxinidir}/requirements-frozen.txt"